diff options
| author | Filipa Lacerda <filipa@gitlab.com> | 2019-06-12 10:28:58 +0100 |
|---|---|---|
| committer | Filipa Lacerda <filipa@gitlab.com> | 2019-06-12 10:28:58 +0100 |
| commit | 0b080f8c8a81c6883582c417be199abc56086208 (patch) | |
| tree | 52657388129959073b52bbb8f72ddfc26a042641 /lib | |
| parent | 093b2f42c4efc1f679ec742e0a28145171127782 (diff) | |
| parent | e206e66bbba9e36a75eb485f65c26f1d5a05e90f (diff) | |
| download | gitlab-ce-0b080f8c8a81c6883582c417be199abc56086208.tar.gz | |
Merge branch 'master' into generate-spans-for-sections
* master: (784 commits)
Update Serverless.gitlab-ci-yml to fix lint warning
Document when milestones and labels links are missing
Update git object deduplication overview
Apply tooltip on truncated commit title
Add link to import video
Update merge options for auto merge strategies
Update note style in various docs
Add link to Auto DevOps demo video
Add link to Groups video
Remove glossary content and make necessary doc edits to clarify terms
Docs for Gitlab Runner docker use of private registries
Metrics reports documentation is incomplete
Update issues/index.md with note on label priority sorting
API issues - minor cleanup of permission check
Fix gl_dropdown scrolling to top on assignee click
Final doc review
Remove docs for a removed ES rake task
Fixed markdown link in Docker compose note
Documented the Admin Area's *Groups* page
Docs: Updating issue data and actions doc
...
Diffstat (limited to 'lib')
185 files changed, 2803 insertions, 971 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f4a96b9711b..20f8c637274 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -98,7 +98,6 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages - mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistry diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb deleted file mode 100644 index da756daadcc..00000000000 --- a/lib/api/circuit_breakers.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module API - class CircuitBreakers < Grape::API - before { authenticated_as_admin! } - - resource :circuit_breakers do - params do - requires :type, - type: String, - desc: "The type of circuitbreaker", - values: ['repository_storage'] - end - resource ':type' do - namespace '', requirements: { type: 'repository_storage' } do - desc 'Get all git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get do - present [] - end - - desc 'Get all failing git storages' do - detail 'This feature was introduced in GitLab 9.5' - end - get 'failing' do - present [] - end - - desc 'Reset all storage failures and open circuitbreaker' do - detail 'This feature was introduced in GitLab 9.5' - end - delete do - end - end - end - end - end -end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 65eb9bfb87e..80913f4ca07 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -96,17 +96,27 @@ module API end end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' end post ':id/repository/commits' do + if params[:start_project] + start_project = find_project!(params[:start_project]) + + unless user_project.forked_from?(start_project) + forbidden!("Project is not included in the fork network for #{start_project.full_name}") + end + end + authorize_push_to_branch!(params[:branch]) attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 5928ee1657b..693172b7d08 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -206,7 +206,7 @@ module API delete_note(noteable, params[:note_id]) end - if Noteable::RESOLVABLE_TYPES.include?(noteable_type.to_s) + if Noteable.resolvable_types.include?(noteable_type.to_s) desc "Resolve/unresolve an existing #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1a3318fe849..880b07d0a84 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -239,6 +239,7 @@ module API end end + expose :empty_repo?, as: :empty_repo expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } @@ -302,6 +303,7 @@ module API expose :commit_count expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end @@ -354,6 +356,7 @@ module API with_options format_with: -> (value) { value.to_i } do expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end @@ -573,6 +576,8 @@ module API expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| issue end + + expose :task_completion_status end class Issue < IssueBasic @@ -721,6 +726,8 @@ module API end expose :squash + + expose :task_completion_status end class MergeRequest < MergeRequestBasic @@ -1303,6 +1310,7 @@ module API class Variable < Grape::Entity expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } + expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } end class Pipeline < PipelineBasic diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 6893c8c40be..ec1020c7c78 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,34 +7,9 @@ module API before { authenticate_non_get! } - helpers do - params :optional_params_ce do - optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, - values: Gitlab::VisibilityLevel.string_values, - default: Gitlab::VisibilityLevel.string_level( - Gitlab::CurrentSettings.current_application_settings.default_group_visibility), - desc: 'The visibility of the group' - optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' - end - - params :optional_params_ee do - end - - params :optional_update_params_ee do - end - end - - include ::API::Helpers::GroupsHelpers + helpers Helpers::GroupsHelpers helpers do - params :optional_params do - use :optional_params_ce - use :optional_params_ee - end - params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 7e4539d0419..00bcf6b055b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -445,7 +445,7 @@ module API end def present_carrierwave_file!(file, supports_direct_download: true) - return not_found! unless file.exists? + return not_found! unless file&.exists? if file.file_storage? present_disk_file!(file.path, file.filename) diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index ae677547760..2c33d79f6c8 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -4,6 +4,30 @@ module API module Helpers module GroupsHelpers extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ce do + optional :description, type: String, desc: 'The description of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' + end + + params :optional_params_ee do + end + + params :optional_update_params_ee do + end + + params :optional_params do + use :optional_params_ce + use :optional_params_ee + end end end end diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index fc66cec5341..5b7199fddb0 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -3,6 +3,14 @@ module API module Helpers module IssuesHelpers + extend Grape::API::Helpers + + params :optional_issue_params_ee do + end + + params :optional_issues_params_ee do + end + def self.update_params_at_least_one_of [ :assignee_id, diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 73d58ee7f37..1395ffadab9 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -19,28 +19,13 @@ module API .non_request end - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_project(project) - shared_group_ids = project.project_group_links.pluck(:group_id) - project_group_ids = project.group&.self_and_ancestors&.pluck(:id) - source_ids = [project.id, project_group_ids, shared_group_ids] - .flatten - .compact - Member.includes(:user) - .joins(user: :project_authorizations) - .where(project_authorizations: { project_id: project.id }) - .where(source_id: source_ids) + MembersFinder.new(project, current_user).execute(include_invited_groups_members: true) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_group(group) - source_ids = group.self_and_ancestors.pluck(:id) - Member.includes(:user) - .where(source_id: source_ids) - .where(source_type: 'Namespace') + GroupMembersFinder.new(group).execute end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 94b58a64d26..2a9b17ad22a 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -23,7 +23,7 @@ module API def base_request_uri @base_request_uri ||= URI.parse(request.url).tap do |uri| uri.host = Gitlab.config.gitlab.host - uri.port = nil + uri.port = Gitlab.config.gitlab.port end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index aaf32dafca4..813e46e9520 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -4,48 +4,45 @@ module API module Helpers module ProjectsHelpers extend ActiveSupport::Concern + extend Grape::API::Helpers - included do - helpers do - params :optional_project_params_ce do - optional :description, type: String, desc: 'The description of the project' - optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' - optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' - optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' - optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' - optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' - optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' - optional :public_builds, type: Boolean, desc: 'Perform public builds' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - optional :tag_list, type: Array[String], desc: 'The list of tags for a project' - optional :avatar, type: File, desc: 'Avatar image for project' - optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' - optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' - optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" - optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' - end + params :optional_project_params_ce do + optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' + optional :avatar, type: File, desc: 'Avatar image for project' + optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' + optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" + optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project' + end + + params :optional_project_params_ee do + end - if Gitlab.ee? - params :optional_project_params_ee do - optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' - optional :approvals_before_merge, type: Integer, desc: 'How many approvers should approve merge request by default' - optional :mirror, type: Boolean, desc: 'Enables pull mirroring in a project' - optional :mirror_trigger_builds, type: Boolean, desc: 'Pull mirroring triggers builds' - end - end + params :optional_project_params do + use :optional_project_params_ce + use :optional_project_params_ee + end + + params :optional_filter_params_ee do + end - params :optional_project_params do - use :optional_project_params_ce - use :optional_project_params_ee if Gitlab.ee? - end - end + params :optional_update_params_ee do end def self.update_params_at_least_one_of diff --git a/lib/api/helpers/protected_branches_helpers.rb b/lib/api/helpers/protected_branches_helpers.rb new file mode 100644 index 00000000000..0fc6841d79a --- /dev/null +++ b/lib/api/helpers/protected_branches_helpers.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Helpers + module ProtectedBranchesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + end + end +end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 953be7f3798..44c577204b8 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -563,6 +563,12 @@ module API name: :notify_only_broken_pipelines, type: Boolean, desc: 'Notify only broken pipelines' + }, + { + required: false, + name: :notify_only_default_branch, + type: Boolean, + desc: 'Send notifications only for the default branch' } ], 'pivotaltracker' => [ diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb new file mode 100644 index 00000000000..6441bb579ff --- /dev/null +++ b/lib/api/helpers/settings_helpers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Helpers + module SettingsHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + + def self.optional_attributes + [*::ApplicationSettingsHelper.visible_attributes, + *::ApplicationSettingsHelper.external_authorization_service_attributes, + :performance_bar_allowed_group_id].freeze + end + end + end +end diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb new file mode 100644 index 00000000000..56fd3c6602d --- /dev/null +++ b/lib/api/helpers/users_helpers.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Helpers + module UsersHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + + params :optional_index_params_ee do + end + end + end +end diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb new file mode 100644 index 00000000000..78a92d0f5a6 --- /dev/null +++ b/lib/api/helpers/variables_helpers.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Helpers + module VariablesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + end + end +end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0b4da01f3c8..039ebf92187 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -9,16 +9,6 @@ module API before { authenticate_non_get! } helpers do - if Gitlab.ee? - params :issues_params_ee do - optional :weight, types: [Integer, String], integer_none_any: true, desc: 'The weight of the issue' - end - - params :issue_params_ee do - optional :weight, type: Integer, desc: 'The weight of the issue' - end - end - params :issues_stats_params do optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' @@ -47,7 +37,7 @@ module API optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' - use :issues_params_ee if Gitlab.ee? + use :optional_issues_params_ee end params :issues_params do @@ -73,7 +63,7 @@ module API optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - use :issue_params_ee if Gitlab.ee? + use :optional_issue_params_ee end end @@ -262,14 +252,9 @@ module API issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - # Setting updated_at only allowed for admins and owners as well - if params[:updated_at].present? - if current_user.admin? || user_project.owner == current_user || current_user.owned_groups.include?(user_project.owner) - issue.system_note_timestamp = params[:updated_at] - else - params.delete(:updated_at) - end - end + # Setting updated_at is allowed only for admins and owners + params.delete(:updated_at) unless current_user.can?(:set_issue_updated_at, user_project) + issue.system_note_timestamp = params[:updated_at] update_params = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index ce85772e4ed..955624404f1 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -386,9 +386,8 @@ module API ) if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) + AutoMergeService.new(merge_request.target_project, current_user, merge_params) + .execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) else ::MergeRequests::MergeService .new(merge_request.target_project, current_user, merge_params) @@ -429,11 +428,9 @@ module API post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do merge_request = find_project_merge_request(params[:merge_request_iid]) - unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) + unauthorized! unless merge_request.can_cancel_auto_merge?(current_user) - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user) - .cancel(merge_request) + AutoMergeService.new(merge_request.target_project, current_user).cancel(merge_request) end desc 'Rebase the merge request against its target branch' do diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index c64ec2fcc95..71891e43dcc 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -3,7 +3,8 @@ module API class ProjectImport < Grape::API include PaginationParams - include Helpers::ProjectsHelpers + + helpers Helpers::ProjectsHelpers helpers do def import_params diff --git a/lib/api/projects.rb b/lib/api/projects.rb index cb0106592f5..1e14c77b5be 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -6,27 +6,12 @@ module API class Projects < Grape::API include PaginationParams include Helpers::CustomAttributes - include Helpers::ProjectsHelpers + + helpers Helpers::ProjectsHelpers before { authenticate_non_get! } helpers do - if Gitlab.ee? - params :optional_filter_params_ee do - optional :wiki_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where wiki checksum is failed' - optional :repository_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where repository checksum is failed' - end - - params :optional_update_params_ee do - optional :mirror_user_id, type: Integer, desc: 'User responsible for all the activity surrounding a pull mirror event' - optional :only_mirror_protected_branches, type: Grape::API::Boolean, desc: 'Only mirror protected branches' - optional :mirror_overwrites_diverged_branches, type: Grape::API::Boolean, desc: 'Pull mirror overwrites diverged branches' - optional :import_url, type: String, desc: 'URL from which the project is imported' - optional :packages_enabled, type: Grape::API::Boolean, desc: 'Enable project packages feature' - optional :fallback_approvals_required, type: Integer, desc: 'Overall approvals required when no rule is present' - end - end - # EE::API::Projects would override this method def apply_filters(projects) projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] @@ -77,7 +62,7 @@ module API optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' - use :optional_filter_params_ee if Gitlab.ee? + use :optional_filter_params_ee end params :create_params do @@ -296,7 +281,7 @@ module API optional :path, type: String, desc: 'The path of the repository' use :optional_project_params - use :optional_update_params_ee if Gitlab.ee? + use :optional_update_params_ee at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index f8cce1ed784..33dea25289a 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -8,6 +8,8 @@ module API before { authorize_admin_project } + helpers Helpers::ProtectedBranchesHelpers + params do requires :id, type: String, desc: 'The ID of a project' end @@ -52,29 +54,7 @@ module API values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' - if Gitlab.ee? - optional :unprotect_access_level, type: Integer, - values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels, - desc: 'Access levels allowed to unprotect (defaults: `40`, maintainer access level)' - - optional :allowed_to_push, type: Array, desc: 'An array of users/groups allowed to push' do - optional :access_level, type: Integer, values: ProtectedBranch::PushAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - - optional :allowed_to_merge, type: Array, desc: 'An array of users/groups allowed to merge' do - optional :access_level, type: Integer, values: ProtectedBranch::MergeAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - - optional :allowed_to_unprotect, type: Array, desc: 'An array of users/groups allowed to unprotect' do - optional :access_level, type: Integer, values: ProtectedBranch::UnprotectAccessLevel.allowed_access_levels - optional :user_id, type: Integer - optional :group_id, type: Integer - end - end + use :optional_params_ee end # rubocop: disable CodeReuse/ActiveRecord post ':id/protected_branches' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 8046acfa397..6767ef882cb 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -4,6 +4,8 @@ module API class Settings < Grape::API before { authenticated_as_admin! } + helpers Helpers::SettingsHelpers + helpers do def current_settings @current_setting ||= @@ -136,54 +138,10 @@ module API desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." end - if Gitlab.ee? - optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch' - - given elasticsearch_aws: ->(val) { val } do - optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key' - requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured' - optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key' - end - - optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing' - - given elasticsearch_indexing: ->(val) { val } do - optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search' - requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")' - optional :elasticsearch_limit_indexing, type: Boolean, desc: 'Limit Elasticsearch to index certain namespaces and projects' - end - - given elasticsearch_limit_indexing: ->(val) { val } do - optional :elasticsearch_namespace_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The namespace ids to index with Elasticsearch.' - optional :elasticsearch_project_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The project ids to index with Elasticsearch.' - end - - optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons' - optional :help_text, type: String, desc: 'GitLab server administrator information' - optional :repository_size_limit, type: Integer, desc: 'Size limit per repository (MB)' - optional :file_template_project_id, type: Integer, desc: 'ID of project where instance-level file templates are stored.' - optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.' - optional :snowplow_enabled, type: Boolean, desc: 'Enable Snowplow' - - given snowplow_enabled: ->(val) { val } do - requires :snowplow_collector_uri, type: String, desc: 'Snowplow Collector URI' - optional :snowplow_cookie_domain, type: String, desc: 'Snowplow cookie domain' - optional :snowplow_site_id, type: String, desc: 'Snowplow Site/Application ID' - end - - optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' - end - - optional_attributes = [*::ApplicationSettingsHelper.visible_attributes, - *::ApplicationSettingsHelper.external_authorization_service_attributes, - :performance_bar_allowed_group_id] - - if Gitlab.ee? - optional_attributes += EE::ApplicationSettingsHelper.possible_licensed_attributes - end + use :optional_params_ee - optional(*optional_attributes) - at_least_one_of(*optional_attributes) + optional(*Helpers::SettingsHelpers.optional_attributes) + at_least_one_of(*Helpers::SettingsHelpers.optional_attributes) end put "application/settings" do attrs = declared_params(include_missing: false) diff --git a/lib/api/users.rb b/lib/api/users.rb index 2f23e33bd4a..6afeebb6890 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -15,6 +15,8 @@ module API authenticate_non_get! end + helpers Helpers::UsersHelpers + helpers do # rubocop: disable CodeReuse/ActiveRecord def find_user_by_id(params) @@ -52,10 +54,7 @@ module API optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider - if Gitlab.ee? - optional :shared_runners_minutes_limit, type: Integer, desc: 'Pipeline minutes quota for this user' - optional :extra_shared_runners_minutes_limit, type: Integer, desc: '(admin-only) Extra pipeline minutes quota for this user' - end + use :optional_params_ee end params :sort_params do @@ -85,10 +84,7 @@ module API use :sort_params use :pagination use :with_custom_attributes - - if Gitlab.ee? - optional :skip_ldap, type: Boolean, default: false, desc: 'Skip LDAP users' - end + use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord get do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index a1bb21b3a06..af1d7936556 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,6 +7,8 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + helpers Helpers::VariablesHelpers + helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain @@ -54,12 +56,11 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - if Gitlab.ee? - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - end + use :optional_params_ee end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -80,12 +81,11 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' - optional :protected, type: String, desc: 'Whether the variable is protected' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - if Gitlab.ee? - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - end + use :optional_params_ee end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb index f346151a3c1..2acc9d13f07 100644 --- a/lib/banzai/commit_renderer.rb +++ b/lib/banzai/commit_renderer.rb @@ -2,7 +2,7 @@ module Banzai module CommitRenderer - ATTRIBUTES = [:description, :title].freeze + ATTRIBUTES = [:description, :title, :full_title].freeze def self.render(commits, project, user = nil) obj_renderer = ObjectRenderer.new(user: user, default_project: project) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 44b151d01e7..0224dd8fcd1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -363,6 +363,14 @@ module Banzai group_ref end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 4d67140b0a1..4892668fc22 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -104,14 +104,6 @@ module Banzai matches[:namespace] && matches[:project] end - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end - - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end - def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index fce042e8946..08969753d75 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -51,13 +51,13 @@ module Banzai # default implementation. return super(text, pattern) if pattern != Milestone.reference_pattern - text.gsub(pattern) do |match| + unescape_html_entities(text).gsub(pattern) do |match| milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone yield match, milestone.id, $~[:project], $~[:namespace], $~ else - match + escape_html_entities(match) end end end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index f4cc8beeb52..77b5053f38c 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -4,6 +4,8 @@ module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter class Rewriter + UNSAFE_SLUG_REGEXES = [/\Ajavascript:/i].freeze + def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) @wiki_base_path = wiki && wiki.wiki_base_path @@ -35,6 +37,8 @@ module Banzai # Of the form `./link`, `../link`, or similar def apply_hierarchical_link_rules! + return if slug_considered_unsafe? + @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' end @@ -54,6 +58,10 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end + + def slug_considered_unsafe? + UNSAFE_SLUG_REGEXES.any? { |r| r.match?(@slug) } + end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 30cafd11834..d67f461be57 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -25,7 +25,6 @@ module Banzai Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, - Filter::EmojiFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, @@ -34,6 +33,7 @@ module Banzai *reference_filters, + Filter::EmojiFilter, Filter::TaskListFilter, Filter::InlineDiffFilter, diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 7db5f5e1f7d..c2da7fec7cc 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -70,8 +70,11 @@ module Banzai # Build the raw <a> tag just with a link as href and content if # it's originally a link pattern. We shouldn't return a plain text href. original_link = - if link_reference == 'true' && href = original_content - %(<a href="#{href}">#{href}</a>) + if link_reference == 'true' + href = node.attr('href') + content = original_content + + %(<a href="#{href}">#{content}</a>) end # The reference should be replaced by the original link's content, diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 3f107fbbf3b..ccaf06c5d6a 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -40,6 +40,7 @@ module Gitlab SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze + HTTP_PROXY_ENV_VARS = %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY).freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com @@ -66,6 +67,10 @@ module Gitlab end end + def self.http_proxy_env? + HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] } + end + def self.process_name return 'sidekiq' if Sidekiq.server? return 'console' if defined?(Rails::Console) diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 36fc8061d92..72a187377d0 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -55,7 +55,7 @@ module Gitlab private def info - auth_hash.info + auth_hash['info'] end def get_info(key) diff --git a/lib/gitlab/auth_logger.rb b/lib/gitlab/auth_logger.rb new file mode 100644 index 00000000000..6d3edba02b0 --- /dev/null +++ b/lib/gitlab/auth_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class AuthLogger < Gitlab::JsonLogger + def self.file_name_noext + 'auth' + end + end +end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb new file mode 100644 index 00000000000..886c41a2b9d --- /dev/null +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CalculateWikiSizes + def perform(start_id, stop_id) + ::ProjectStatistics.where(wiki_size: nil) + .where(id: start_id..stop_id) + .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| + statistics.refresh!(only: [:wiki_size]) + rescue => e + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb new file mode 100644 index 00000000000..0e93b2cb2fa --- /dev/null +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # save validity time pages domain + class FillValidTimeForPagesDomainCertificate + # define PagesDomain with only needed code + class PagesDomain < ActiveRecord::Base + self.table_name = 'pages_domains' + + def x509 + return unless certificate.present? + + @x509 ||= OpenSSL::X509::Certificate.new(certificate) + rescue OpenSSL::X509::CertificateError + nil + end + end + + def perform(start_id, stop_id) + PagesDomain.where(id: start_id..stop_id).find_each do |domain| + if Gitlab::Database.mysql? + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before, + certificate_valid_not_after: domain.x509&.not_after + ) + else + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) + end + rescue => e + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + end + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb new file mode 100644 index 00000000000..447fec8903c --- /dev/null +++ b/lib/gitlab/background_migration/reset_merge_status.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Updates the range of given MRs to merge_status "unchecked", if they're opened + # and mergeable. + class ResetMergeStatus + def perform(from_id, to_id) + relation = MergeRequest.where(id: from_id..to_id, + state: 'opened', + merge_status: 'can_be_merged') + + relation.update_all(merge_status: 'unchecked') + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 0beeb44c272..21c42857895 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -24,7 +24,7 @@ module Gitlab end entry :ports, Entry::Ports, - description: 'Ports used expose the image' + description: 'Ports used to expose the image' attributes :ports diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 290c9591b98..762532f7007 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -34,7 +34,7 @@ module Gitlab message: 'should be on_success, on_failure, ' \ 'always, manual or delayed' } validates :dependencies, array_of_strings: true - validates :extends, type: String + validates :extends, array_of_strings_or_string: true end validates :start_in, duration: { limit: '1 day' }, if: :delayed? diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 084fa4047a4..8d16371e857 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -24,6 +24,9 @@ module Gitlab validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } end + entry :ports, Entry::Ports, + description: 'Ports used to expose the service' + def alias value[:alias] end diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb index 7793db09d33..0001a259281 100644 --- a/lib/gitlab/ci/config/extendable/entry.rb +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -5,6 +5,8 @@ module Gitlab class Config class Extendable class Entry + include Gitlab::Utils::StrongMemoize + InvalidExtensionError = Class.new(Extendable::ExtensionError) CircularDependencyError = Class.new(Extendable::ExtensionError) NestingTooDeepError = Class.new(Extendable::ExtensionError) @@ -28,34 +30,46 @@ module Gitlab end def value - @value ||= @context.fetch(@key) + strong_memoize(:value) do + @context.fetch(@key) + end end - def base_hash! - @base ||= Extendable::Entry - .new(extends_key, @context, self) - .extend! + def base_hashes! + strong_memoize(:base_hashes) do + extends_keys.map do |key| + Extendable::Entry + .new(key, @context, self) + .extend! + end + end end - def extends_key - value.fetch(:extends).to_s.to_sym if extensible? + def extends_keys + strong_memoize(:extends_keys) do + next unless extensible? + + Array(value.fetch(:extends)).map(&:to_s).map(&:to_sym) + end end def ancestors - @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key) + strong_memoize(:ancestors) do + Array(@parent&.ancestors) + Array(@parent&.key) + end end def extend! return value unless extensible? - if unknown_extension? + if unknown_extensions.any? raise Entry::InvalidExtensionError, - "#{key}: unknown key in `extends`" + "#{key}: unknown keys in `extends` (#{show_keys(unknown_extensions)})" end - if invalid_base? + if invalid_bases.any? raise Entry::InvalidExtensionError, - "#{key}: invalid base hash in `extends`" + "#{key}: invalid base hashes in `extends` (#{show_keys(invalid_bases)})" end if nesting_too_deep? @@ -68,11 +82,18 @@ module Gitlab "#{key}: circular dependency detected in `extends`" end - @context[key] = base_hash!.deep_merge(value) + merged = {} + base_hashes!.each { |h| merged.deep_merge!(h) } + + @context[key] = merged.deep_merge!(value) end private + def show_keys(keys) + keys.join(', ') + end + def nesting_too_deep? ancestors.count > MAX_NESTING_LEVELS end @@ -81,12 +102,16 @@ module Gitlab ancestors.include?(key) end - def unknown_extension? - !@context.key?(extends_key) + def unknown_extensions + strong_memoize(:unknown_extensions) do + extends_keys.reject { |key| @context.key?(key) } + end end - def invalid_base? - !@context[extends_key].is_a?(Hash) + def invalid_bases + strong_memoize(:invalid_bases) do + extends_keys.reject { |key| @context[key].is_a?(Hash) } + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb new file mode 100644 index 00000000000..54a0e2ad9dd --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class And < Lexeme::Operator + PATTERN = /&&/.freeze + + def evaluate(variables = {}) + @left.evaluate(variables) && @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + + def self.precedence + 11 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 70c774416f6..7ebd2e25398 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -15,10 +15,14 @@ module Gitlab end def self.scan(scanner) - if scanner.scan(self::PATTERN) + if scanner.scan(pattern) Expression::Token.new(scanner.matched, self) end end + + def self.pattern + self::PATTERN + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 668e85f5b9e..62f4c14f597 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -8,11 +8,6 @@ module Gitlab class Equals < Lexeme::Operator PATTERN = /==/.freeze - def initialize(left, right) - @left = left - @right = right - end - def evaluate(variables = {}) @left.evaluate(variables) == @right.evaluate(variables) end @@ -20,6 +15,10 @@ module Gitlab def self.build(_value, behind, ahead) new(behind, ahead) end + + def self.precedence + 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index cd17bc4d78b..ecfab627226 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -8,21 +8,36 @@ module Gitlab class Matches < Lexeme::Operator PATTERN = /=~/.freeze - def initialize(left, right) - @left = left - @right = right - end - def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) regexp.scan(text.to_s).any? + + if ci_variables_complex_expressions? + # return offset of first match, or nil if no matches + if match = regexp.scan(text.to_s).first + text.to_s.index(match) + end + else + # return true or false + regexp.scan(text.to_s).any? + end end def self.build(_value, behind, ahead) new(behind, ahead) end + + def self.precedence + 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end + + private + + def ci_variables_complex_expressions? + Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb index 5fcc9406cc8..8166bcd5730 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -8,11 +8,6 @@ module Gitlab class NotEquals < Lexeme::Operator PATTERN = /!=/.freeze - def initialize(left, right) - @left = left - @right = right - end - def evaluate(variables = {}) @left.evaluate(variables) != @right.evaluate(variables) end @@ -20,6 +15,10 @@ module Gitlab def self.build(_value, behind, ahead) new(behind, ahead) end + + def self.precedence + 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 14544d33e25..831c27fa0ea 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -8,11 +8,6 @@ module Gitlab class NotMatches < Lexeme::Operator PATTERN = /\!~/.freeze - def initialize(left, right) - @left = left - @right = right - end - def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) @@ -23,6 +18,10 @@ module Gitlab def self.build(_value, behind, ahead) new(behind, ahead) end + + def self.precedence + 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index 3ebceb92eb7..3ddab7800c8 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -6,9 +6,29 @@ module Gitlab module Expression module Lexeme class Operator < Lexeme::Base + # This operator class is design to handle single operators that take two + # arguments. Expression::Parser was originally designed to read infix operators, + # and so the two operands are called "left" and "right" here. If we wish to + # implement an Operator that takes a greater or lesser number of arguments, a + # structural change or additional Operator superclass will likely be needed. + + OperatorError = Class.new(Expression::ExpressionError) + + def initialize(left, right) + raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate + raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate + + @left = left + @right = right + end + def self.type :operator end + + def self.precedence + raise NotImplementedError + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb new file mode 100644 index 00000000000..807876f905a --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Or < Lexeme::Operator + PATTERN = /\|\|/.freeze + + def evaluate(variables = {}) + @left.evaluate(variables) || @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + + def self.precedence + 12 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 2b719c9c6fc..e4cf360a1c1 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -8,10 +8,11 @@ module Gitlab require_dependency 're2' class Pattern < Lexeme::Value - PATTERN = %r{^/.+/[ismU]*$}.freeze + PATTERN = %r{^/.+/[ismU]*$}.freeze + NEW_PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = regexp + @value = self.class.eager_matching_with_escape_characters? ? regexp.gsub(/\\\//, '/') : regexp unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -24,9 +25,17 @@ module Gitlab raise Expression::RuntimeError, 'Invalid regular expression!' end + def self.pattern + eager_matching_with_escape_characters? ? NEW_PATTERN : PATTERN + end + def self.build(string) new(string) end + + def self.eager_matching_with_escape_characters? + Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index e14edfae51d..22c210ae26b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -20,6 +20,19 @@ module Gitlab Expression::Lexeme::NotMatches ].freeze + NEW_LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Pattern, + Expression::Lexeme::Null, + Expression::Lexeme::Equals, + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches, + Expression::Lexeme::And, + Expression::Lexeme::Or + ].freeze + MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) @@ -45,7 +58,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = LEXEMES.find do |type| + lexeme = available_lexemes.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end @@ -58,6 +71,10 @@ module Gitlab raise Lexer::SyntaxError, 'Too many tokens!' end + + def available_lexemes + Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) ? NEW_LEXEMES : LEXEMES + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index ed184309ab4..589bf32a4d7 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -5,17 +5,30 @@ module Gitlab module Pipeline module Expression class Parser + ParseError = Class.new(Expression::ExpressionError) + def initialize(tokens) @tokens = tokens.to_enum @nodes = [] end - ## - # This produces a reverse descent parse tree. - # - # It currently does not support precedence of operators. - # def tree + if Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) + rpn_parse_tree + else + reverse_descent_parse_tree + end + end + + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + + private + + # This produces a reverse descent parse tree. + # It does not support precedence of operators. + def reverse_descent_parse_tree while token = @tokens.next case token.type when :operator @@ -32,8 +45,51 @@ module Gitlab @nodes.last || Lexeme::Null.new end - def self.seed(statement) - new(Expression::Lexer.new(statement).tokens) + def rpn_parse_tree + results = [] + + tokens_rpn.each do |token| + case token.type + when :value + results.push(token.build) + when :operator + right_operand = results.pop + left_operand = results.pop + + token.build(left_operand, right_operand).tap do |res| + results.push(res) + end + else + raise ParseError, 'Unprocessable token found in parse tree' + end + end + + raise ParseError, 'Unreachable nodes in parse tree' if results.count > 1 + raise ParseError, 'Empty parse tree' if results.count < 1 + + results.pop + end + + # Parse the expression into Reverse Polish Notation + # (See: Shunting-yard algorithm) + def tokens_rpn + output = [] + operators = [] + + @tokens.each do |token| + case token.type + when :value + output.push(token) + when :operator + if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence + output.push(operators.pop) + end + + operators.push(token) + end + end + + output.concat(operators.reverse) end end end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index ab5ae9caeea..0e81e1bd34c 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -7,27 +7,6 @@ module Gitlab class Statement StatementError = Class.new(Expression::ExpressionError) - GRAMMAR = [ - # presence matchers - %w[variable], - - # positive matchers - %w[variable equals string], - %w[variable equals variable], - %w[variable equals null], - %w[string equals variable], - %w[null equals variable], - %w[variable matches pattern], - - # negative matchers - %w[variable notequals string], - %w[variable notequals variable], - %w[variable notequals null], - %w[string notequals variable], - %w[null notequals variable], - %w[variable notmatches pattern] - ].freeze - def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) @variables = variables.with_indifferent_access @@ -36,10 +15,6 @@ module Gitlab def parse_tree raise StatementError if @lexer.lexemes.empty? - unless GRAMMAR.find { |syntax| syntax == @lexer.lexemes } - raise StatementError, 'Unknown pipeline expression!' - end - Expression::Parser.new(@lexer.tokens).tree end @@ -54,6 +29,7 @@ module Gitlab end def valid? + evaluate parse_tree.is_a?(Lexeme::Base) rescue Expression::ExpressionError false diff --git a/lib/gitlab/ci/templates/.yamllint b/lib/gitlab/ci/templates/.yamllint new file mode 100644 index 00000000000..669c8646ff2 --- /dev/null +++ b/lib/gitlab/ci/templates/.yamllint @@ -0,0 +1,5 @@ +extends: default + +rules: + line-length: disable + document-start: disable diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 120272200c6..2ca6e73a803 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -54,7 +54,7 @@ ensureContainer: - rm -f android-signing-keystore.jks || true artifacts: paths: - - app/build/outputs + - app/build/outputs buildDebug: extends: .build_job @@ -116,6 +116,6 @@ promoteProduction: # We only allow production promotion on `master` because # it has its own production scoped secret variables only: - - master + - master script: - bundle exec fastlane promote_beta_to_production diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index 2be7f933462..b7194110002 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -6,7 +6,7 @@ image: openjdk:8-jdk variables: ANDROID_COMPILE_SDK: "28" ANDROID_BUILD_TOOLS: "28.0.2" - ANDROID_SDK_TOOLS: "4333796" + ANDROID_SDK_TOOLS: "4333796" before_script: - apt-get --quiet update --yes @@ -35,7 +35,7 @@ assembleDebug: - ./gradlew assembleDebug artifacts: paths: - - app/build/outputs/ + - app/build/outputs/ debugTests: stage: test diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 939112e6e41..65a6630365d 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,8 +50,8 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.9 - HELM_VERSION: 2.13.1 + KUBERNETES_VERSION: 1.11.10 + HELM_VERSION: 2.14.0 DOCKER_DRIVER: overlay2 @@ -60,7 +60,7 @@ variables: stages: - build - test - - deploy # dummy stage to follow the template guidelines + - deploy # dummy stage to follow the template guidelines - review - dast - staging @@ -89,4 +89,4 @@ include: dast: except: refs: - - master
\ No newline at end of file + - master diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index 9a8fa9d7091..33a2a534508 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -14,10 +14,10 @@ build: artifacts: paths: - mybinary - # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time - # cache: - # paths: - # - "*.o" + # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time + # cache: + # paths: + # - "*.o" # run tests using the binary built before test: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 1e14aa8bea9..5f17c93b853 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -23,31 +23,31 @@ stages: foodcritic: stage: lint script: - - chef exec foodcritic . + - chef exec foodcritic . cookstyle: stage: lint script: - - chef exec cookstyle . + - chef exec cookstyle . chefspec: stage: test script: - - chef exec rspec spec + - chef exec rspec spec # Set up your test matrix here. Example: -#verify-centos-6: -# stage: functional -# before_script: -# - apt-get update -# - apt-get -y install rsync -# script: -# - kitchen verify default-centos-6 --destroy=always +# verify-centos-6: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-6 --destroy=always # -#verify-centos-7: -# stage: functional -# before_script: -# - apt-get update -# - apt-get -y install rsync -# script: -# - kitchen verify default-centos-7 --destroy=always +# verify-centos-7: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-7 --destroy=always diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0610cb9ccc0..c3568c0d2c8 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -8,7 +8,7 @@ before_script: # If you need to install any external applications, like a # postgres client, you may want to uncomment the line below: # - #- apt-get update -y + # - apt-get update -y # # Retrieve project dependencies # Do this on before_script since it'll be shared between both test and @@ -17,6 +17,6 @@ before_script: test: script: - # If you need to run any migrations or configure the database, this - # would be the point to do it. - - lein test + # If you need to run any migrations or configure the database, this + # would be the point to do it. + - lein test diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 36386a19fdc..e9301a2638d 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -22,15 +22,15 @@ cache: # services such as redis or postgres before_script: - apt-get update -qq && apt-get install -y -qq libxml2-dev - - crystal -v # Print out Crystal version for debugging + - crystal -v # Print out Crystal version for debugging - shards # If you are using built-in Crystal Spec. spec: script: - - crystal spec + - crystal spec # If you are using minitest.cr minitest: script: - - crystal test/spec_test.cr # change to the file(s) you execute for tests + - crystal test/spec_test.cr # change to the file(s) you execute for tests diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index 1d8be6f017e..d35fcb0f807 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -16,19 +16,19 @@ variables: # http://docs.gitlab.com/ce/ci/yaml/README.html#cache cache: paths: - - ~/.cache/pip/ + - ~/.cache/pip/ # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging # Uncomment next line if your Django app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - pip install -r requirements.txt # To get Django tests to work you may need to create a settings file using # the following DATABASES: -# +# # DATABASES = { # 'default': { # 'ENGINE': 'django.db.backends.postgresql_psycopg2', @@ -46,4 +46,4 @@ test: variables: DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" script: - - python manage.py test + - python manage.py test diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index cf9c731637c..4d4c6a64cd5 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -15,4 +15,4 @@ before_script: mix: script: - - mix test + - mix test diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 55fda1a4799..1b686bc6cc0 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -16,21 +16,21 @@ before_script: - cd $GOPATH/src/$REPO_NAME stages: - - test - - build - - deploy + - test + - build + - deploy format: - stage: test - script: - - go fmt $(go list ./... | grep -v /vendor/) - - go vet $(go list ./... | grep -v /vendor/) - - go test -race $(go list ./... | grep -v /vendor/) + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) compile: - stage: build - script: - - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary - artifacts: - paths: - - mybinary + stage: build + script: + - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + artifacts: + paths: + - mybinary diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index dbc868238f8..efcd1d3ddc0 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -16,25 +16,25 @@ variables: # We use SDKMan as tool for managing versions before_script: - - apt-get update -qq && apt-get install -y -qq unzip - - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh - - sdk install gradle $GRADLE_VERSION < /dev/null - - sdk use gradle $GRADLE_VERSION -# As it's not a good idea to version gradle.properties feel free to add your -# environments variable here - - echo grailsVersion=$GRAILS_VERSION > gradle.properties - - echo gradleWrapperVersion=2.14 >> gradle.properties -# refresh dependencies from your project - - ./gradlew --refresh-dependencies -# Be aware that if you are using Angular profile, -# Bower cannot be run as root if you don't allow it before. -# Feel free to remove next line if you are not using Bower - - echo {\"allow_root\":true} > /root/.bowerrc + - apt-get update -qq && apt-get install -y -qq unzip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install gradle $GRADLE_VERSION < /dev/null + - sdk use gradle $GRADLE_VERSION + # As it's not a good idea to version gradle.properties feel free to add your + # environments variable here + - echo grailsVersion=$GRAILS_VERSION > gradle.properties + - echo gradleWrapperVersion=2.14 >> gradle.properties + # refresh dependencies from your project + - ./gradlew --refresh-dependencies + # Be aware that if you are using Angular profile, + # Bower cannot be run as root if you don't allow it before. + # Feel free to remove next line if you are not using Bower + - echo {\"allow_root\":true} > /root/.bowerrc # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). build: - script: - - ./gradlew build + script: + - ./gradlew build diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 546c4affb4e..a09217e8cf0 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -26,8 +26,8 @@ performance: - mv sitespeed-results/data/performance.json performance.json artifacts: paths: - - performance.json - - sitespeed-results/ + - performance.json + - sitespeed-results/ only: refs: - branches diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 779f4b5f006..1d55c64ec56 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -432,6 +432,7 @@ rollout 100%: --set postgresql.postgresUser="$POSTGRES_USER" \ --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set postgresql.imageTag="$POSTGRES_VERSION" \ --set application.migrateCommand="$DB_MIGRATE" \ $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ @@ -507,23 +508,13 @@ rollout 100%: kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } - # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN - function ensure_kube_ingress_base_domain() { - if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then - export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN - fi - } - function check_kube_domain() { - ensure_kube_ingress_base_domain - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then echo "In order to deploy or use Review Apps," - echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" echo "or by defining a variable at group or project level." echo "You can also manually add it in .gitlab-ci.yml" - echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" false else true diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 2c4683fbfbb..32d4e07d398 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -20,7 +20,7 @@ # want coverage results. - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' # Comment out below if you do not want coverage results. - - julia -e 'using Pkg; Pkg.add("Coverage"); + - julia -e 'using Pkg; Pkg.add("Coverage"); import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); using Coverage; cl, tl = get_summary(process_folder()); println("(", cl/tl*100, "%) covered")' @@ -55,17 +55,16 @@ pages: image: julia:0.7 stage: deploy script: - - apt-get update -qq && apt-get install -y git # needed by Documenter - - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure - - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter - - julia --color=yes docs/make.jl # make documentation - - mv docs/build public # move to the directory picked up by Gitlab pages + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages artifacts: paths: - public only: - - master - + - master # WARNING: This template is using the `julia` images from [Docker # Hub][3]. One can use custom Julia images and/or the official ones found diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index e1cd29ecc94..9bde04dff19 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -16,8 +16,8 @@ variables: # http://docs.gitlab.com/ce/ci/yaml/README.html#cache cache: paths: - - vendor/ - - node_modules/ + - vendor/ + - node_modules/ # This is a basic example for a gem or script which doesn't use # services such as redis or postgres @@ -37,7 +37,7 @@ before_script: - docker-php-ext-enable xdebug # Install Composer and project dependencies. - curl -sS https://getcomposer.org/installer | php - - php composer.phar install + - php composer.phar install # Install Node dependencies. # comment this out if you don't have a node dependency - npm install @@ -63,10 +63,10 @@ before_script: test: script: - # run laravel tests - - php vendor/bin/phpunit --coverage-text --colors=never - # run frontend tests - # if you have any task for testing frontend - # set it in your package.json script - # comment this out if you don't have a frontend test - - npm test + # run laravel tests + - php vendor/bin/phpunit --coverage-text --colors=never + # run frontend tests + # if you have any task for testing frontend + # set it in your package.json script + # comment this out if you don't have a frontend test + - npm test diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index c9838c7a7ff..13ab98d3a16 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,18 +1,14 @@ ---- +# This file is a template, and might need editing before it works on your project. + # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # # For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html -# -# This template will build and test your projects as well as create the documentation. -# + +# This template will build and test your projects # * Caches downloaded dependencies and plugins between invocation. # * Verify but don't deploy merge requests. # * Deploy built artifacts from master branch only. -# * Shows how to use multiple jobs in test stage for verifying functionality -# with multiple JDKs. -# * Uses site:stage to collect the documentation for multi-module projects. -# * Publishes the documentation for `master` branch. variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. @@ -23,78 +19,38 @@ variables: # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" +# This template uses jdk8 for verifying and deploying images +image: maven:3.3.9-jdk-8 + # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' cache: paths: - .m2/repository -# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. -# Because some enforcer rules might check dependency convergence and class duplications -# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. -.validate: &validate - stage: build - script: - - 'mvn $MAVEN_CLI_OPTS test-compile' - # For merge requests do not `deploy` but only run `verify`. # See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html .verify: &verify stage: test script: - - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + - 'mvn $MAVEN_CLI_OPTS verify' except: - master -# Validate merge requests using JDK7 -validate:jdk7: - <<: *validate - image: maven:3.3.9-jdk-7 - -# Validate merge requests using JDK8 -validate:jdk8: - <<: *validate - image: maven:3.3.9-jdk-8 - -# Verify merge requests using JDK7 -verify:jdk7: - <<: *verify - image: maven:3.3.9-jdk-7 - # Verify merge requests using JDK8 verify:jdk8: <<: *verify - image: maven:3.3.9-jdk-8 +# To deploy packages from CI, create a ci_settings.xml file +# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for more details. +# Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. # For `master` branch run `mvn deploy` automatically. -# Here you need to decide whether you want to use JDK7 or 8. -# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. -# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. -# See https://maven.apache.org/settings.html deploy:jdk8: - # Use stage test here, so the pages job may later pickup the created site. - stage: test - script: - - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' - only: - - master - # Archive up the built documentation site. - artifacts: - paths: - - target/staging - image: maven:3.3.9-jdk-8 - -pages: - image: busybox:latest stage: deploy script: - # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, - # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. - - mv target/staging public - dependencies: - - deploy:jdk8 - artifacts: - paths: - - public + - if [ ! -f ci_settings.xml ]; + then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for instructions."; + fi + - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: - master diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 41de1458582..b87178141a1 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -14,14 +14,14 @@ services: # http://docs.gitlab.com/ce/ci/yaml/README.html#cache cache: paths: - - node_modules/ + - node_modules/ test_async: script: - - npm install - - node ./specs/start.js ./specs/async.spec.js + - npm install + - node ./specs/start.js ./specs/async.spec.js test_db: script: - - npm install - - node ./specs/start.js ./specs/db-postgres.spec.js + - npm install + - node ./specs/start.js ./specs/db-postgres.spec.js diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 61a925e0d2d..65abee1f5eb 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,9 +1,9 @@ image: ayufan/openshift-cli stages: - - build # dummy stage to follow the template guidelines + - build # dummy stage to follow the template guidelines - test - - deploy # dummy stage to follow the template guidelines + - deploy # dummy stage to follow the template guidelines - review - staging - production diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 33f44ee9222..b9fee2d5731 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -4,19 +4,19 @@ image: php:7.1.1 # Select what we should cache between builds cache: paths: - - vendor/ + - vendor/ before_script: -- apt-get update -yqq -- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -# Install PHP extensions -- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache -# Install & enable Xdebug for code coverage reports -- pecl install xdebug -- docker-php-ext-enable xdebug -# Install and run Composer -- curl -sS https://getcomposer.org/installer | php -- php composer.phar install + - apt-get update -yqq + - apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev + # Install PHP extensions + - docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache + # Install & enable Xdebug for code coverage reports + - pecl install xdebug + - docker-php-ext-enable xdebug + # Install and run Composer + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install # Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service # See http://docs.gitlab.com/ce/ci/services/README.html for examples. @@ -33,4 +33,4 @@ variables: # If Xdebug was installed you can generate a coverage report and see code coverage metrics. test: script: - - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index d6de8cab5d1..d2dd3fbfb75 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -4,12 +4,12 @@ image: node:4.2.2 pages: cache: paths: - - node_modules/ + - node_modules/ script: - - npm install -g brunch - - brunch build --production + - npm install -g brunch + - brunch build --production artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index 791afdd23f1..ba422c08614 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -3,11 +3,11 @@ image: alpine pages: script: - - apk update && apk add doxygen - - doxygen doxygen/Doxyfile - - mv doxygen/documentation/html/ public/ + - apk update && apk add doxygen + - doxygen doxygen/Doxyfile + - mv doxygen/documentation/html/ public/ artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 9df2a4797b2..a683561a455 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -4,14 +4,14 @@ image: node:latest # http://docs.gitlab.com/ce/ci/yaml/README.html#cache cache: paths: - - node_modules/ + - node_modules/ pages: script: - - yarn install - - ./node_modules/.bin/gatsby build --prefix-paths + - yarn install + - ./node_modules/.bin/gatsby build --prefix-paths artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 249a168aa33..92f25280c6e 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -2,11 +2,11 @@ pages: stage: deploy script: - - mkdir .public - - cp -r * .public - - mv .public public + - mkdir .public + - cp -r * .public + - mv .public public artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 4b58003ee10..0e206423fa5 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -4,12 +4,12 @@ image: node:4.2.2 pages: cache: paths: - - node_modules + - node_modules script: - - npm install -g harp - - harp compile ./ public + - npm install -g harp + - harp compile ./ public artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index a9e195370f7..d91a8d7421f 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -3,15 +3,15 @@ image: node:10.15.3 pages: script: - - npm install hexo-cli -g - - test -e package.json && npm install - - hexo generate + - npm install hexo-cli -g + - test -e package.json && npm install + - hexo generate artifacts: paths: - - public + - public cache: paths: - node_modules key: project only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index f9ddcc6fb0a..9a3ecd1c34f 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -3,15 +3,15 @@ image: dettmering/hugo-build pages: script: - - hugo + - hugo artifacts: paths: - - public + - public only: - - master + - master test: script: - - hugo + - hugo except: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index f5b40f2b9f1..7a441a2f70f 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -3,7 +3,7 @@ image: python:2.7 cache: paths: - - vendor/ + - vendor/ test: stage: test @@ -20,6 +20,6 @@ pages: - hyde gen -d public artifacts: paths: - - public + - public only: - master diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 7a485f8d135..886b6c36249 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -4,28 +4,28 @@ # JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers # # This yml works with jBake 2.5.1 -# Feel free to change JBAKE_VERSION version +# Feel free to change JBAKE_VERSION version # # HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ image: java:8 variables: - JBAKE_VERSION: 2.5.1 + JBAKE_VERSION: 2.5.1 # We use SDKMan as tool for managing versions before_script: - - apt-get update -qq && apt-get install -y -qq unzip zip - - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh - - sdk install jbake $JBAKE_VERSION < /dev/null - - sdk use jbake $JBAKE_VERSION + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION # This build job produced the output directory of your site pages: - script: - - jbake . public - artifacts: - paths: - - public + script: + - jbake . public + artifacts: + paths: + - public diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 37f50554036..0d742aa282d 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -6,25 +6,24 @@ variables: JEKYLL_ENV: production before_script: -- bundle install + - bundle install test: stage: test script: - - bundle exec jekyll build -d test + - bundle exec jekyll build -d test artifacts: paths: - - test + - test except: - - master + - master pages: stage: deploy script: - - bundle exec jekyll build -d public + - bundle exec jekyll build -d public artifacts: paths: - - public + - public only: - - master - + - master diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 5ca4619e200..2d26b86a328 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -7,8 +7,8 @@ image: php:7.2 # These folders are cached between builds cache: paths: - - vendor/ - - node_modules/ + - vendor/ + - node_modules/ before_script: # Update packages @@ -28,10 +28,10 @@ before_script: pages: script: - - npm run production - - mv build_production public + - npm run production + - mv build_production public artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index c5c44a5d86c..93ab8e0be0d 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -3,10 +3,10 @@ image: python:2.7 pages: script: - - pip install lektor - - lektor build --output-path public + - pip install lektor + - lektor build --output-path public artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index c6ded272150..6524405133a 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -4,13 +4,13 @@ image: node:4.2.2 pages: cache: paths: - - node_modules/ + - node_modules/ script: - - npm install -g metalsmith - - npm install - - make build + - npm install -g metalsmith + - npm install + - make build artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 9f4cc0574d6..57ac323dfdf 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -3,25 +3,25 @@ image: ruby:2.3 cache: paths: - - vendor + - vendor test: script: - - apt-get update -yqqq - - apt-get install -y nodejs - - bundle install --path vendor - - bundle exec middleman build + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build except: - master pages: script: - - apt-get update -yqqq - - apt-get install -y nodejs - - bundle install --path vendor - - bundle exec middleman build + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b469b316ba5..7f037b5f5cf 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -3,10 +3,10 @@ image: ruby:2.3 pages: script: - - bundle install -j4 - - nanoc + - bundle install -j4 + - nanoc artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4762ec9acfd..6d912a89bc1 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -3,13 +3,13 @@ image: ruby:2.3 pages: script: - - apt-get update -qq && apt-get install -qq nodejs - - bundle install -j4 - - bundle exec rake generate - - mv public .public - - mv .public/octopress public + - apt-get update -qq && apt-get install -qq nodejs + - bundle install -j4 + - bundle exec rake generate + - mv public .public + - mv .public/octopress public artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index c5f3154f587..09c6649fc13 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -3,8 +3,8 @@ image: python:2.7-alpine pages: script: - - pip install -r requirements.txt - - pelican -s publishconf.py + - pip install -r requirements.txt + - pelican -s publishconf.py artifacts: paths: - - public/ + - public/ diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 3eaed4e91cd..00b8b94b574 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -18,34 +18,34 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate test: script: - - python setup.py test - - pip install tox flake8 # you can also use tox - - tox -e py36,flake8 + - python setup.py test + - pip install tox flake8 # you can also use tox + - tox -e py36,flake8 run: script: - - python setup.py bdist_wheel - # an alternative approach is to install and run: - - pip install dist/* - # run the command here + - python setup.py bdist_wheel + # an alternative approach is to install and run: + - pip install dist/* + # run the command here artifacts: paths: - - dist/*.whl + - dist/*.whl pages: script: - - pip install sphinx sphinx-rtd-theme - - cd doc ; make html - - mv build/html/ ../public/ + - pip install sphinx sphinx-rtd-theme + - cd doc ; make html + - mv build/html/ ../public/ artifacts: paths: - - public + - public only: - - master + - master diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 93196dbd475..b3cad8b858a 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -21,7 +21,7 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - ruby -v # Print out ruby version for debugging + - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby @@ -29,19 +29,19 @@ before_script: # Optional - Delete if not using `rubocop` rubocop: script: - - rubocop + - rubocop rspec: script: - - rspec spec + - rspec spec rails: variables: DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" script: - - rails db:migrate - - rails db:seed - - rails test + - rails db:migrate + - rails db:seed + - rails test # This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk # are supported too: https://github.com/travis-ci/dpl @@ -49,5 +49,5 @@ deploy: type: deploy environment: production script: - - gem install dpl - - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY + - gem install dpl + - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index cab087c48c7..a25dc38e4e7 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -5,19 +5,19 @@ image: "rust:latest" # Optional: Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. # Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service -#services: -# - mysql:latest -# - redis:latest -# - postgres:latest +# services: +# - mysql:latest +# - redis:latest +# - postgres:latest # Optional: Install a C compiler, cmake and git into the container. # You will often need this when you (or any of your dependencies) depends on C code. -#before_script: -#- apt-get update -yqq -#- apt-get install -yqq --no-install-recommends build-essential +# before_script: +# - apt-get update -yqq +# - apt-get install -yqq --no-install-recommends build-essential # Use cargo to test the project test:cargo: script: - - rustc --version && cargo --version # Print version info for debugging - - cargo test --all --verbose + - rustc --version && cargo --version # Print version info for debugging + - cargo test --all --verbose diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 324e39c7747..5372ec6cceb 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -23,6 +23,9 @@ container_scanning: DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 + ## Disable the proxy for clair-local-scan, otherwise Container Scanning will + ## fail when a proxy is used. + NO_PROXY: ${DOCKER_SERVICE},localhost allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 27a498b2daf..4b55ffd3771 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -15,22 +15,22 @@ dast: image: name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" variables: - # URL to scan: - # DAST_WEBSITE: https://example.com/ - # - # Time limit for target availability (scan is attempted even when timeout): - # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 - # - # Set these variables to scan with an authenticated user: - # DAST_AUTH_URL: https://example.com/sign-in - # DAST_USERNAME: john.doe@example.com - # DAST_PASSWORD: john-doe-password - # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form - # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form - # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between - # - # Perform ZAP Full Scan, which includes both passive and active scanning: - # DAST_FULL_SCAN_ENABLED: "true" + # URL to scan: + # DAST_WEBSITE: https://example.com/ + # + # Time limit for target availability (scan is attempted even when timeout): + # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 + # + # Set these variables to scan with an authenticated user: + # DAST_AUTH_URL: https://example.com/sign-in + # DAST_USERNAME: john.doe@example.com + # DAST_PASSWORD: john-doe-password + # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form + # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form + # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between + # + # Perform ZAP Full Scan, which includes both passive and active scanning: + # DAST_FULL_SCAN_ENABLED: "true" allow_failure: true script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml index 0208beb35b8..b6555e627a1 100644 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/project/merge_requests/license_management.html variables: - LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. license_management: stage: test diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 706692e063b..abf16e5b2e7 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -40,6 +40,7 @@ sast: SAST_BRAKEMAN_LEVEL \ SAST_GOSEC_LEVEL \ SAST_FLAWFINDER_LEVEL \ + SAST_GITLEAKS_ENTROPY_LEVEL \ SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index 0fb7c57ab72..a3db2705bf6 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,25 +8,13 @@ stages: - deploy .serverless:build:image: - variables: - DOCKERFILE: "Dockerfile" stage: build - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - only: - refs: - - master - script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKERFILE --destination $CI_REGISTRY_IMAGE + image: registry.gitlab.com/gitlab-org/gitlabktl:latest + script: /usr/bin/gitlabktl app build .serverless:deploy:image: stage: deploy - image: gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 - only: - refs: - - master + image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development script: - echo "$CI_REGISTRY_IMAGE" diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index 9aa4abd4791..ffed7a0fec2 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -25,7 +25,7 @@ archive_project: - master artifacts: paths: - - build/ProjectName.ipa + - build/ProjectName.ipa tags: - ios_11-3 - xcode_9-3 diff --git a/lib/gitlab/ci/templates/dotNET-Core.yml b/lib/gitlab/ci/templates/dotNET-Core.yml index 708b75f83e8..40ca296d7bd 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.yml @@ -1,4 +1,5 @@ -# This is a simple example illustrating how to build and test .NET Core project +--- +# This is a simple example illustrating how to build and test .NET Core project # with GitLab Continuous Integration / Continuous Delivery. # ### Specify the Docker image @@ -17,21 +18,21 @@ image: microsoft/dotnet:latest # ### Define variables # variables: - # 1) Name of directory where restore and build objects are stored. - OBJECTS_DIRECTORY: 'obj' - # 2) Name of directory used for keeping restored dependencies. - NUGET_PACKAGES_DIRECTORY: '.nuget' - # 3) A relative path to the source code from project repository root. - # NOTE: Please edit this path so it matches the structure of your project! - SOURCE_CODE_PATH: '*/*/' + # 1) Name of directory where restore and build objects are stored. + OBJECTS_DIRECTORY: 'obj' + # 2) Name of directory used for keeping restored dependencies. + NUGET_PACKAGES_DIRECTORY: '.nuget' + # 3) A relative path to the source code from project repository root. + # NOTE: Please edit this path so it matches the structure of your project! + SOURCE_CODE_PATH: '*/*/' # ### Define stage list # # In this example there are only two stages. # Initially, the project will be built and then tested. stages: - - build - - test + - build + - test # ### Define global cache rule # @@ -50,31 +51,31 @@ stages: # With global cache rule, cached dependencies will be downloaded before every job # and then unpacked to the paths as specified below. cache: - # Per-stage and per-branch caching. - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - # Specify three paths that should be cached: + # Per-stage and per-branch caching. + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + # Specify three paths that should be cached: + # + # 1) Main JSON file holding information about package dependency tree, packages versions, + # frameworks etc. It also holds information where to the dependencies were restored. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/project.assets.json' + # 2) Other NuGet and MSBuild related files. Also needed. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/*.csproj.nuget.*' + # 3) Path to the directory where restored dependencies are kept. + - '$NUGET_PACKAGES_DIRECTORY' # - # 1) Main JSON file holding information about package dependency tree, packages versions, - # frameworks etc. It also holds information where to the dependencies were restored. - - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/project.assets.json' - # 2) Other NuGet and MSBuild related files. Also needed. - - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/*.csproj.nuget.*' - # 3) Path to the directory where restored dependencies are kept. - - '$NUGET_PACKAGES_DIRECTORY' - # - # 'pull-push' policy means that latest cache will be downloaded (if it exists) - # before executing the job, and a newer version will be uploaded afterwards. - # Such a setting saves time when there are no changes in referenced third-party - # packages. - # - # For example, if you run a pipeline with changes in your code, - # but with no changes within third-party packages which your project is using, - # then project restore will happen quickly as all required dependencies - # will already be there — unzipped from cache. + # 'pull-push' policy means that latest cache will be downloaded (if it exists) + # before executing the job, and a newer version will be uploaded afterwards. + # Such a setting saves time when there are no changes in referenced third-party + # packages. + # + # For example, if you run a pipeline with changes in your code, + # but with no changes within third-party packages which your project is using, + # then project restore will happen quickly as all required dependencies + # will already be there — unzipped from cache. - # 'pull-push' policy is the default cache policy, you do not have to specify it explicitly. - policy: pull-push + # 'pull-push' policy is the default cache policy, you do not have to specify it explicitly. + policy: pull-push # ### Restore project dependencies # @@ -87,30 +88,30 @@ cache: # # Learn more about GitLab cache: https://docs.gitlab.com/ee/ci/caching/index.html before_script: - - 'dotnet restore --packages $NUGET_PACKAGES_DIRECTORY' + - 'dotnet restore --packages $NUGET_PACKAGES_DIRECTORY' build: - stage: build - # ### Build all projects discovered from solution file. - # - # Note: this will fail if you have any projects in your solution that are not - # .NET Core-based projects (e.g. WCF service), which is based on .NET Framework, - # not .NET Core. In this scenario, you will need to build every .NET Core-based - # project by explicitly specifying a relative path to the directory - # where it is located (e.g. 'dotnet build ./src/ConsoleApp'). - # Only one project path can be passed as a parameter to 'dotnet build' command. - script: - - 'dotnet build --no-restore' + stage: build + # ### Build all projects discovered from solution file. + # + # Note: this will fail if you have any projects in your solution that are not + # .NET Core-based projects (e.g. WCF service), which is based on .NET Framework, + # not .NET Core. In this scenario, you will need to build every .NET Core-based + # project by explicitly specifying a relative path to the directory + # where it is located (e.g. 'dotnet build ./src/ConsoleApp'). + # Only one project path can be passed as a parameter to 'dotnet build' command. + script: + - 'dotnet build --no-restore' tests: - stage: test - # ### Run the tests - # - # You can either run tests for all test projects that are defined in your solution - # with 'dotnet test' or run tests only for specific project by specifying - # a relative path to the directory where it is located (e.g. 'dotnet test ./test/UnitTests'). - # - # You may want to define separate testing jobs for different types of testing - # (e.g. integration tests, unit tests etc). - script: - - 'dotnet test --no-restore' + stage: test + # ### Run the tests + # + # You can either run tests for all test projects that are defined in your solution + # with 'dotnet test' or run tests only for specific project by specifying + # a relative path to the directory where it is located (e.g. 'dotnet test ./test/UnitTests'). + # + # You may want to define separate testing jobs for different types of testing + # (e.g. integration tests, unit tests etc). + script: + - 'dotnet test --no-restore' diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 5e128b793d0..b29f45323f5 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -3,12 +3,12 @@ # (located in gitlab-runner.exe directory): # shell = "powershell" # -# The script is composed of 3 stages: build, test and deploy. +# The script is composed of 3 stages: build, test and deploy. # # The build stage restores NuGet packages and uses msbuild to build the exe and msi # One major issue you'll find is that you can't build msi projects from command line # if you use vdproj. There are workarounds building msi via devenv, but they rarely work -# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# The best solution is migrating your vdproj projects to WiX, as it can be build directly # by msbuild. # # The test stage runs nunit from command line against Test project inside your solution @@ -38,28 +38,28 @@ stages: build_job: stage: build only: - - tags # the build process will only be started by git tag commits + - tags # the build process will only be started by git tag commits script: - - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies - - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project artifacts: - expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on paths: - - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder - - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder - - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests test_job: stage: test only: - tags script: - - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests artifacts: - when: always # save test results even when the task fails - expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + when: always # save test results even when the task fails + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on paths: - - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder dependencies: - build_job @@ -79,7 +79,6 @@ deploy_job: - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' - 'xcopy /y ".\TestResult.xml" "$deployFolder"' - dependencies: - build_job - test_job diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index a4a9e96c1d2..87aea8527d1 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,13 +1,13 @@ # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ -# You will also need fastlane and signing configuration for this to work, along with a MacOS runner. +# You will also need fastlane and signing configuration for this to work, along with a MacOS runner. # These details are provided in the blog post. # Note that when you're using the shell executor for MacOS builds, the # build and tests run as the identity of the runner logged in user, directly on # the build host. This is less secure than using container executors, so please -# take a look at our security implications documentation at -# https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional +# take a look at our security implications documentation at +# https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional # detail on what to keep in mind in this scenario. stages: @@ -27,4 +27,4 @@ build: - bundle exec fastlane build artifacts: paths: - - ./*.ipa + - ./*.ipa diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b05dca409d1..e0f9eb59924 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -44,6 +44,14 @@ module Gitlab (@master_restart_hooks ||= []) << block end + def on_master_start(&block) + if in_clustered_environment? + on_before_fork(&block) + else + on_worker_start(&block) + end + end + # # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 4ed9a9a02ab..4affc52b7b0 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -27,6 +27,9 @@ module Gitlab # is restarted already, thus periodically restarting workers shouldn't be # needed. config.rolling_restart_frequency = false + + observer = Gitlab::Cluster::PumaWorkerKillerObserver.new + config.pre_term = observer.callback end PumaWorkerKiller.start diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb new file mode 100644 index 00000000000..3b4ebc3fbae --- /dev/null +++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + class PumaWorkerKillerObserver + def initialize + @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller') + end + + # returns the Proc to be used as the observer callback block + def callback + method(:log_termination) + end + + private + + def log_termination(worker) + labels = { worker: "worker_#{worker.index}" } + + @counter.increment(labels) + end + end + end +end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index f0ca397609d..7a0fb419f8e 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -1,6 +1,4 @@ # frozen_string_literal: true -require 'net/http' -require 'json' require_relative 'teammate' @@ -8,7 +6,6 @@ module Gitlab module Danger module Helper RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - ROULETTE_DATA_URL = URI.parse('https://about.gitlab.com/roulette.json').freeze # Returns a list of all files that have been added, modified or renamed. # `git.modified_files` might contain paths that already have been renamed, @@ -49,32 +46,6 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end - # Looks up the current list of GitLab team members and parses it into a - # useful form - # - # @return [Array<Teammate>] - def team - @team ||= - begin - rsp = Net::HTTP.get_response(ROULETTE_DATA_URL) - raise "Failed to read #{ROULETTE_DATA_URL}: #{rsp.code} #{rsp.message}" unless - rsp.is_a?(Net::HTTPSuccess) - - data = JSON.parse(rsp.body) - data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } - rescue JSON::ParserError - raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" - end - end - - # Like +team+, but only returns teammates in the current project, based on - # project_name. - # - # @return [Array<Teammate>] - def project_team - team.select { |member| member.in_project?(project_name) } - end - # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| @@ -101,7 +72,8 @@ module Gitlab CATEGORY_LABELS = { docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", - qa: "~QA" + qa: "~QA", + test: "~test for `spec/features/*`" }.freeze CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -133,6 +105,7 @@ module Gitlab %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb new file mode 100644 index 00000000000..062eda38ee4 --- /dev/null +++ b/lib/gitlab/danger/roulette.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' +require 'cgi' + +require_relative 'teammate' + +module Gitlab + module Danger + module Roulette + ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json' + HTTPError = Class.new(RuntimeError) + + # Looks up the current list of GitLab team members and parses it into a + # useful form + # + # @return [Array<Teammate>] + def team + @team ||= + begin + data = http_get_json(ROULETTE_DATA_URL) + data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } + rescue JSON::ParserError + raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" + end + end + + # Like +team+, but only returns teammates in the current project, based on + # project_name. + # + # @return [Array<Teammate>] + def project_team(project_name) + team.select { |member| member.in_project?(project_name) } + end + + def canonical_branch_name(branch_name) + branch_name.gsub(/^[ce]e-|-[ce]e$/, '') + end + + def new_random(seed) + Random.new(Digest::MD5.hexdigest(seed).to_i(16)) + end + + # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the + # selection will change on next spin + def spin_for_person(people, random:) + person = nil + people = people.dup + + people.size.times do + person = people.sample(random: random) + + break person unless out_of_office?(person) + + people -= [person] + end + + person + end + + private + + def out_of_office?(person) + username = CGI.escape(person.username) + api_endpoint = "https://gitlab.com/api/v4/users/#{username}/status" + response = http_get_json(api_endpoint) + response["message"]&.match?(/OOO/i) + rescue HTTPError, JSON::ParserError + false # this is no worse than not checking for OOO + end + + def http_get_json(url) + rsp = Net::HTTP.get_response(URI.parse(url)) + + unless rsp.is_a?(Net::HTTPSuccess) + raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" + end + + JSON.parse(rsp.body) + end + end + end +end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index bfada512727..b44f134f2c1 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,11 +3,12 @@ module Gitlab module Danger class Teammate - attr_reader :name, :username, :projects + attr_reader :name, :username, :role, :projects def initialize(options = {}) - @name = options['name'] @username = options['username'] + @name = options['name'] || @username + @role = options['role'] @projects = options['projects'] end @@ -20,20 +21,32 @@ module Gitlab end # Traintainers also count as reviewers - def reviewer?(project, category) - capabilities(project).include?("reviewer #{category}") || traintainer?(project, category) + def reviewer?(project, category, labels) + has_capability?(project, category, :reviewer, labels) || + traintainer?(project, category, labels) end - def traintainer?(project, category) - capabilities(project).include?("trainee_maintainer #{category}") + def traintainer?(project, category, labels) + has_capability?(project, category, :trainee_maintainer, labels) end - def maintainer?(project, category) - capabilities(project).include?("maintainer #{category}") + def maintainer?(project, category, labels) + has_capability?(project, category, :maintainer, labels) end private + def has_capability?(project, category, kind, labels) + case category + when :test + area = role[/Test Automation Engineer, (\w+)/, 1] + + area && labels.any?(area) if kind == :reviewer + else + capabilities(project).include?("#{kind} #{category}") + end + end + def capabilities(project) Array(projects.fetch(project, [])) end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 65601dcdf31..16e62622ed4 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -44,6 +44,7 @@ module Gitlab data[:commit] = build_data_for_commit(project, user, note) elsif note.for_issue? data[:issue] = note.noteable.hook_attrs + data[:issue][:labels] = note.noteable.labels(&:hook_attrs) elsif note.for_merge_request? data[:merge_request] = note.noteable.hook_attrs elsif note.for_snippet? diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index fa06fb935f7..e1e813849bf 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -19,7 +19,7 @@ module Gitlab def hook_attrs(pipeline) { id: pipeline.id, - ref: pipeline.ref, + ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, before_sha: pipeline.before_sha, diff --git a/lib/gitlab/diff/suggestion.rb b/lib/gitlab/diff/suggestion.rb index 027c7a31bcf..4a3ac2106e2 100644 --- a/lib/gitlab/diff/suggestion.rb +++ b/lib/gitlab/diff/suggestion.rb @@ -33,6 +33,8 @@ module Gitlab end def to_content + return "" if @text.blank? + # The parsed suggestion doesn't have information about the correct # ending characters (we may have a line break, or not), so we take # this information from the last line being changed (last diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb index c8c03d5d001..6e17ffaf6ff 100644 --- a/lib/gitlab/diff/suggestions_parser.rb +++ b/lib/gitlab/diff/suggestions_parser.rb @@ -10,10 +10,12 @@ module Gitlab # Returns an array of Gitlab::Diff::Suggestion which represents each # suggestion in the given text. # - def parse(text, position:, project:) + def parse(text, position:, project:, supports_suggestion: true) return [] unless position.complete? - html = Banzai.render(text, project: nil, no_original_data: true) + html = Banzai.render(text, project: nil, + no_original_data: true, + suggestions_filter_enabled: supports_suggestion) doc = Nokogiri::HTML(html) suggestion_nodes = doc.search('pre.suggestion') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 455588f3c66..8a2e711ec4e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -34,34 +34,6 @@ module Gitlab TagExistsError = Class.new(StandardError) ChecksumError = Class.new(StandardError) - class << self - def create_hooks(repo_path, global_hooks_path) - local_hooks_path = File.join(repo_path, 'hooks') - real_local_hooks_path = :not_found - - begin - real_local_hooks_path = File.realpath(local_hooks_path) - rescue Errno::ENOENT - # real_local_hooks_path == :not_found - end - - # Do nothing if hooks already exist - unless real_local_hooks_path == File.realpath(global_hooks_path) - if File.exist?(local_hooks_path) - # Move the existing hooks somewhere safe - FileUtils.mv( - local_hooks_path, - "#{local_hooks_path}.old.#{Time.now.to_i}") - end - - # Create the hooks symlink - FileUtils.ln_sf(global_hooks_path, local_hooks_path) - end - - true - end - end - # Directory name of repo attr_reader :name @@ -310,6 +282,11 @@ module Gitlab (size.to_f / 1024).round(2) end + # Return git object directory size in bytes + def object_directory_size + gitaly_repository_client.get_object_directory_size.to_f * 1024 + end + # Build an array of commits. # # Usage. diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 3f13ebeb9d0..dfff6823689 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -5,12 +5,15 @@ module Gitlab module GitRefValidator extend self + + EXPANDED_PREFIXES = %w[refs/heads/ refs/remotes/].freeze + DISALLOWED_PREFIXES = %w[-].freeze + # Validates a given name against the git reference specification # # Returns true for a valid reference name, false otherwise def validate(ref_name) - not_allowed_prefixes = %w(refs/heads/ refs/remotes/ -) - return false if ref_name.start_with?(*not_allowed_prefixes) + return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) return false if ref_name == 'HEAD' begin @@ -19,5 +22,21 @@ module Gitlab return false end end + + def validate_merge_request_branch(ref_name) + return false if ref_name.start_with?(*DISALLOWED_PREFIXES) + + expanded_name = if ref_name.start_with?(*EXPANDED_PREFIXES) + ref_name + else + "refs/heads/#{ref_name}" + end + + begin + Rugged::Reference.valid_name?(expanded_name) + rescue ArgumentError + return false + end + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 05e06eec012..e683d4e5bbe 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,7 +33,7 @@ module Gitlab SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze # Server feature flags should use '_' to separate words. - SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE, 'delta_islands'].freeze + SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE].freeze MUTEX = Mutex.new diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 077b63205a8..d16e45c964d 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -65,9 +65,9 @@ module Gitlab our_commit_oid: @our_commit_oid, target_repository: target_repository.gitaly_repository, their_commit_oid: @their_commit_oid, - source_branch: source_branch, - target_branch: target_branch, - commit_message: resolution.commit_message, + source_branch: encode_binary(source_branch), + target_branch: encode_binary(target_branch), + commit_message: encode_binary(resolution.commit_message), user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly ) end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index e4a59ee3f9b..b42e6cbad8d 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -107,7 +107,7 @@ module Gitlab branch: encode_binary(branch), target_ref: encode_binary(target_ref), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - message: message + message: encode_binary(message) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 68b17e86608..d8e9dccb644 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -47,6 +47,13 @@ module Gitlab response.size end + def get_object_directory_size + request = Gitaly::GetObjectDirectorySizeRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) + + response.size + end + def apply_gitattributes(revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 9d81441d96e..1d3541b80c7 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -29,29 +29,13 @@ module Gitlab end def execute - jid = generate_jid - - # The original import JID is the JID of the RepositoryImportWorker job, - # which will be removed once that job completes. Reusing that JID could - # result in StuckImportJobsWorker marking the job as stuck before we get - # to running Stage::ImportRepositoryWorker. - # - # We work around this by setting the JID to a custom generated one, then - # refreshing it in the various stages whenever necessary. - Gitlab::SidekiqStatus - .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - - project.import_state.update_column(:jid, jid) + Gitlab::Import::SetAsyncJid.set_jid(project) Stage::ImportRepositoryWorker .perform_async(project.id) true end - - def generate_jid - "github-importer/#{project.id}" - end end end end diff --git a/lib/gitlab/graphql/loaders/batch_commit_loader.rb b/lib/gitlab/graphql/loaders/batch_commit_loader.rb new file mode 100644 index 00000000000..f410c3195f8 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_commit_loader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchCommitLoader + def initialize(repository, blob_id) + @repository, @blob_id = repository, blob_id + end + + def find + BatchLoader.for(blob_id).batch(key: repository) do |blob_ids, loader, batch_args| + Gitlab::Git::Blob.batch_lfs_pointers(batch_args[:key], blob_ids).each do |loaded_blob| + loader.call(loaded_blob.id, loaded_blob.lfs_oid) + end + end + end + + private + + attr_reader :repository, :blob_id + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 5a0099dc6b1..50d3293fcbb 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -12,7 +12,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find - BatchLoader.for({ model: model_class, id: model_id }).batch do |loader_info, loader| + BatchLoader.for({ model: model_class, id: model_id.to_i }).batch do |loader_info, loader| per_model = loader_info.group_by { |info| info[:model] } per_model.each do |model, info| ids = info.map { |i| i[:id] } diff --git a/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb b/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb new file mode 100644 index 00000000000..5e151f4dbd7 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_project_statistics_loader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchProjectStatisticsLoader + attr_reader :project_id + + def initialize(project_id) + @project_id = project_id + end + + def find + BatchLoader.for(project_id).batch do |project_ids, loader| + ProjectStatistics.for_project_ids(project_ids).each do |statistics| + loader.call(statistics.project_id, statistics) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb b/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb deleted file mode 100644 index 95db130dbfc..00000000000 --- a/lib/gitlab/graphql/query_analyzers/log_query_complexity.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module QueryAnalyzers - class LogQueryComplexity - class << self - def analyzer - GraphQL::Analysis::QueryComplexity.new do |query, complexity| - # temporary until https://gitlab.com/gitlab-org/gitlab-ce/issues/59587 - Rails.logger.info("[GraphQL Query Complexity] #{complexity} | admin? #{query.context[:current_user]&.admin?}") - end - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb new file mode 100644 index 00000000000..01b55a1667f --- /dev/null +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module QueryAnalyzers + class LoggerAnalyzer + COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } + DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } + + def analyze?(query) + Feature.enabled?(:graphql_logging, default_enabled: true) + end + + def initial_value(query) + variables = process_variables(query.provided_variables) + default_initial_values(query).merge({ + query_string: query.query_string, + variables: variables + }) + rescue => e + Gitlab::Sentry.track_exception(e) + default_initial_values(query) + end + + def call(memo, visit_type, irep_node) + memo + end + + def final_value(memo) + return if memo.nil? + + analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER] + complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers) + + memo[:depth] = depth + memo[:complexity] = complexity + memo[:duration] = duration(memo[:time_started]).round(1) + + GraphqlLogger.info(memo.except!(:time_started, :query)) + rescue => e + Gitlab::Sentry.track_exception(e) + end + + private + + def process_variables(variables) + if variables.respond_to?(:to_s) + variables.to_s + else + variables + end + end + + def duration(time_started) + nanoseconds = Gitlab::Metrics::System.monotonic_time - time_started + nanoseconds * 1000000 + end + + def default_initial_values(query) + { + time_started: Gitlab::Metrics::System.monotonic_time, + query_string: nil, + query: query, + variables: nil, + duration: nil + } + end + end + end + end +end diff --git a/lib/gitlab/graphql/representation/tree_entry.rb b/lib/gitlab/graphql/representation/tree_entry.rb new file mode 100644 index 00000000000..7ea83db5876 --- /dev/null +++ b/lib/gitlab/graphql/representation/tree_entry.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Representation + class TreeEntry < SimpleDelegator + class << self + def decorate(entries, repository) + return if entries.nil? + + entries.map do |entry| + if entry.is_a?(TreeEntry) + entry + else + self.new(entry, repository) + end + end + end + end + + attr_accessor :repository + + def initialize(raw_entry, repository) + @repository = repository + + super(raw_entry) + end + end + end + end +end diff --git a/lib/gitlab/graphql_logger.rb b/lib/gitlab/graphql_logger.rb new file mode 100644 index 00000000000..43d917908b6 --- /dev/null +++ b/lib/gitlab/graphql_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class GraphqlLogger < Gitlab::JsonLogger + def self.file_name_noext + 'graphql_json' + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index d39ff8c21cc..cfc9ebe4f92 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -44,7 +44,8 @@ module Gitlab human_total_time_spent: issue.human_total_time_spent, human_time_estimate: issue.human_time_estimate, assignee_ids: issue.assignee_ids, - assignee_id: issue.assignee_ids.first # This key is deprecated + assignee_id: issue.assignee_ids.first, # This key is deprecated + labels: issue.labels } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index bcd9e2be35f..db2b4dde244 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -9,9 +9,16 @@ module Gitlab BlockedUrlError = Class.new(StandardError) RedirectionTooDeep = Class.new(StandardError) + HTTP_ERRORS = [ + SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, + Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, + Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, + Gitlab::HTTP::RedirectionTooDeep + ].freeze + include HTTParty # rubocop:disable Gitlab/HTTParty - connection_adapter ProxyHTTPConnectionAdapter + connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) super diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index a64cb47e77e..41eab3658bc 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -10,17 +10,19 @@ # # This option will take precedence over the global setting. module Gitlab - class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter + class HTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - unless allow_local_requests? - begin - Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) - rescue Gitlab::UrlBlocker::BlockedUrlError => e - raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" - end + begin + @uri, hostname = Gitlab::UrlBlocker.validate!(uri, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection?) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" end - super + super.tap do |http| + http.hostname_override = hostname if hostname + end end private @@ -29,6 +31,12 @@ module Gitlab options.fetch(:allow_local_requests, allow_settings_local_requests?) end + def dns_rebind_protection? + return false if Gitlab.http_proxy_env? + + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end + def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? end diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb new file mode 100644 index 00000000000..584d24045ee --- /dev/null +++ b/lib/gitlab/import/set_async_jid.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# The original import JID is the JID of the RepositoryImportWorker job, +# which will be removed once that job completes. Reusing that JID could +# result in StuckImportJobsWorker marking the job as stuck before we get +# to running Stage::ImportRepositoryWorker. +# +# We work around this by setting the JID to a custom generated one, then +# refreshing it in the various stages whenever necessary. +module Gitlab + module Import + module SetAsyncJid + def self.set_jid(project) + jid = generate_jid(project) + + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.import_state.update_column(:jid, jid) + end + + def self.generate_jid(project) + "async-import/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 93b37b7bc5f..c28a1674018 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class AttributeCleaner ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_html\Z/).freeze def self.clean(*args) new(*args).clean @@ -24,7 +25,11 @@ module Gitlab private def prohibited_key?(key) - key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + key =~ PROHIBITED_REFERENCES && !permitted_key?(key) + end + + def permitted_key?(key) + ALLOWED_REFERENCES.include?(key) end def excluded_key?(key) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 7bbcb53f016..71c44af9254 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -156,6 +156,9 @@ excluded_attributes: - :when - :artifacts_file - :artifacts_metadata + - :artifacts_file_store + - :artifacts_metadata_store + - :artifacts_size - :commands push_event_payload: - :event_id diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6be95a16513..a154de5419e 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -59,7 +59,11 @@ module Gitlab end def member_hash(member) - parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + parsed_hash(member).merge( + 'source_id' => @project.id, + 'importing' => true, + 'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min + ).except('user_id') end def parsed_hash(member) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index e1e70a008d9..efd3f550a22 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -153,6 +153,9 @@ module Gitlab @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') @relation_hash.delete('commands') + @relation_hash.delete('artifacts_file_store') + @relation_hash.delete('artifacts_metadata_store') + @relation_hash.delete('artifacts_size') imported_object elsif @relation_name == :merge_requests diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 67952ca0f2d..e4d625b5738 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), - ImportSource.new('manifest', 'Manifest file', nil) + ImportSource.new('manifest', 'Manifest file', nil), + ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer) ].freeze class << self diff --git a/lib/gitlab/kubernetes/errors.rb b/lib/gitlab/kubernetes/errors.rb new file mode 100644 index 00000000000..81bf636eef7 --- /dev/null +++ b/lib/gitlab/kubernetes/errors.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Errors + CONNECTION = [ + SocketError, + OpenSSL::SSL::SSLError, + Errno::ECONNRESET, + Errno::ENETUNREACH, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Net::OpenTimeout, + Net::ReadTimeout, + IPAddr::InvalidAddressError + ].freeze + + AUTHENTICATION = [ + OpenSSL::X509::CertificateError + ].freeze + end + end +end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index ff1dadf9247..978cafae9ac 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -24,6 +24,7 @@ module Gitlab def uninstall(command) namespace.ensure_exists! + create_config_map(command) delete_pod!(command.pod_name) kubeclient.create_pod(command.pod_resource) diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb index 6a7f5e965c5..136268c974b 100644 --- a/lib/gitlab/lets_encrypt/challenge.rb +++ b/lib/gitlab/lets_encrypt/challenge.rb @@ -7,7 +7,7 @@ module Gitlab @acme_challenge = acme_challenge end - delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge + delegate :token, :file_content, :status, :request_validation, to: :acme_challenge private diff --git a/lib/gitlab/lets_encrypt/client.rb b/lib/gitlab/lets_encrypt/client.rb index d7468b06767..66aea137012 100644 --- a/lib/gitlab/lets_encrypt/client.rb +++ b/lib/gitlab/lets_encrypt/client.rb @@ -3,6 +3,8 @@ module Gitlab module LetsEncrypt class Client + include Gitlab::Utils::StrongMemoize + PRODUCTION_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory' STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' @@ -35,6 +37,8 @@ module Gitlab def enabled? return false unless Feature.enabled?(:pages_auto_ssl) + return false unless private_key + Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted end @@ -45,7 +49,11 @@ module Gitlab end def private_key - @private_key ||= OpenSSL::PKey.read(Gitlab::Application.secrets.lets_encrypt_private_key) + strong_memoize(:private_key) do + private_key_string = Gitlab::CurrentSettings.lets_encrypt_private_key + private_key_string ||= generate_private_key + OpenSSL::PKey.read(private_key_string) if private_key_string + end end def admin_email @@ -69,6 +77,19 @@ module Gitlab STAGING_DIRECTORY_URL end end + + def generate_private_key + return if Gitlab::Database.read_only? + + application_settings = Gitlab::CurrentSettings.current_application_settings + application_settings.with_lock do + unless application_settings.lets_encrypt_private_key + application_settings.update(lets_encrypt_private_key: OpenSSL::PKey::RSA.new(4096).to_pem) + end + + application_settings.lets_encrypt_private_key + end + end end end end diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb index 5109b5e9843..9c2365f98a8 100644 --- a/lib/gitlab/lets_encrypt/order.rb +++ b/lib/gitlab/lets_encrypt/order.rb @@ -13,7 +13,16 @@ module Gitlab ::Gitlab::LetsEncrypt::Challenge.new(challenge) end - delegate :url, :status, to: :acme_order + def request_certificate(domain:, private_key:) + csr = ::Acme::Client::CertificateRequest.new( + private_key: OpenSSL::PKey.read(private_key), + subject: { common_name: domain } + ) + + acme_order.finalize(csr: csr) + end + + delegate :url, :status, :expires, :certificate, to: :acme_order private diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb new file mode 100644 index 00000000000..0354c710a3f --- /dev/null +++ b/lib/gitlab/markdown_cache.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module MarkdownCache + # Increment this number every time the renderer changes its output + CACHE_COMMONMARK_VERSION_START = 10 + CACHE_COMMONMARK_VERSION = 16 + + BaseError = Class.new(StandardError) + UnsupportedClassError = Class.new(BaseError) + end +end diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb new file mode 100644 index 00000000000..f3abe631779 --- /dev/null +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module MarkdownCache + module ActiveRecord + module Extension + extend ActiveSupport::Concern + + included do + # Using before_update here conflicts with elasticsearch-model somehow + before_create :refresh_markdown_cache, if: :invalidated_markdown_cache? + before_update :refresh_markdown_cache, if: :invalidated_markdown_cache? + end + + # Always exclude _html fields from attributes (including serialization). + # They contain unredacted HTML, which would be a security issue + def attributes + attrs = super + html_fields = cached_markdown_fields.html_fields + whitelisted = cached_markdown_fields.html_fields_whitelisted + exclude_fields = html_fields - whitelisted + + attrs.except!(*exclude_fields) + attrs.delete('cached_markdown_version') if whitelisted.empty? + + attrs + end + + def changed_markdown_fields + changed_attributes.keys.map(&:to_s) & cached_markdown_fields.markdown_fields.map(&:to_s) + end + + def write_markdown_field(field_name, value) + write_attribute(field_name, value) + end + + def markdown_field_changed?(field_name) + attribute_changed?(field_name) + end + + def save_markdown(updates) + return unless persisted? && Gitlab::Database.read_write? + + update_columns(updates) + end + end + end + end +end diff --git a/lib/gitlab/markdown_cache/field_data.rb b/lib/gitlab/markdown_cache/field_data.rb new file mode 100644 index 00000000000..14622c0f186 --- /dev/null +++ b/lib/gitlab/markdown_cache/field_data.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module MarkdownCache + # Knows about the relationship between markdown and html field names, and + # stores the rendering contexts for the latter + class FieldData + def initialize + @data = {} + end + + delegate :[], :[]=, to: :@data + + def markdown_fields + @data.keys + end + + def html_field(markdown_field) + "#{markdown_field}_html" + end + + def html_fields + @html_fields ||= markdown_fields.map { |field| html_field(field) } + end + + def html_fields_whitelisted + markdown_fields.each_with_object([]) do |field, fields| + if @data[field].fetch(:whitelisted, false) + fields << html_field(field) + end + end + end + end + end +end diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb new file mode 100644 index 00000000000..97fc23343b4 --- /dev/null +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module MarkdownCache + module Redis + module Extension + extend ActiveSupport::Concern + + attr_reader :cached_markdown_version + + class_methods do + def cache_markdown_field(markdown_field, context = {}) + super + + # define the `[field]_html` accessor + html_field = cached_markdown_fields.html_field(markdown_field) + define_method(html_field) do + load_cached_markdown unless markdown_data_loaded? + + instance_variable_get("@#{html_field}") + end + end + end + + private + + def save_markdown(updates) + markdown_store.save(updates) + end + + def write_markdown_field(field_name, value) + instance_variable_set("@#{field_name}", value) + end + + def markdown_field_changed?(field_name) + false + end + + def changed_markdown_fields + [] + end + + def cached_markdown + @cached_data ||= markdown_store.read + end + + def load_cached_markdown + cached_markdown.each do |field_name, value| + write_markdown_field(field_name, value) + end + end + + def markdown_data_loaded? + cached_markdown_version.present? || markdown_store.loaded? + end + + def markdown_store + @store ||= Gitlab::MarkdownCache::Redis::Store.new(self) + end + end + end + end +end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb new file mode 100644 index 00000000000..0f954404808 --- /dev/null +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module MarkdownCache + module Redis + class Store + EXPIRES_IN = 1.day + + def initialize(subject) + @subject = subject + @loaded = false + end + + def save(updates) + @loaded = false + + Gitlab::Redis::Cache.with do |r| + r.mapped_hmset(markdown_cache_key, updates) + r.expire(markdown_cache_key, EXPIRES_IN) + end + end + + def read + @loaded = true + + results = Gitlab::Redis::Cache.with do |r| + r.mapped_hmget(markdown_cache_key, *fields) + end + # The value read from redis is a string, so we're converting it back + # to an int. + results[:cached_markdown_version] = results[:cached_markdown_version].to_i + results + end + + def loaded? + @loaded + end + + private + + def fields + @fields ||= @subject.cached_markdown_fields.html_fields + [:cached_markdown_version] + end + + def markdown_cache_key + unless @subject.respond_to?(:cache_key) + raise Gitlab::MarkdownCache::UnsupportedClassError, + "This class has no cache_key to use for caching" + end + + "markdown_cache:#{@subject.cache_key}" + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb index 94aabd0466c..4664aee71f6 100644 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ b/lib/gitlab/metrics/dashboard/base_service.rb @@ -6,13 +6,13 @@ module Gitlab module Metrics module Dashboard class BaseService < ::BaseService - DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError + PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError def get_dashboard return error("#{dashboard_path} could not be found.", :not_found) unless path_available? success(dashboard: process_dashboard) - rescue DASHBOARD_LAYOUT_ERROR => e + rescue PROCESSING_ERROR => e error(e.message, :unprocessable_entity) end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb index dd986020693..a33a010ad97 100644 --- a/lib/gitlab/metrics/dashboard/processor.rb +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -11,11 +11,13 @@ module Gitlab SYSTEM_SEQUENCE = [ Stages::CommonMetricsInserter, Stages::ProjectMetricsInserter, + Stages::EndpointInserter, Stages::Sorter ].freeze PROJECT_SEQUENCE = [ Stages::CommonMetricsInserter, + Stages::EndpointInserter, Stages::Sorter ].freeze diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index a6d1f974556..0db7b176e8d 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,7 +5,8 @@ module Gitlab module Dashboard module Stages class BaseStage - DashboardLayoutError = Class.new(StandardError) + DashboardProcessingError = Class.new(StandardError) + LayoutError = Class.new(DashboardProcessingError) DEFAULT_PANEL_TYPE = 'area-chart' @@ -25,15 +26,15 @@ module Gitlab protected def missing_panel_groups! - raise DashboardLayoutError.new('Top-level key :panel_groups must be an array') + raise LayoutError.new('Top-level key :panel_groups must be an array') end def missing_panels! - raise DashboardLayoutError.new('Each "panel_group" must define an array :panels') + raise LayoutError.new('Each "panel_group" must define an array :panels') end def missing_metrics! - raise DashboardLayoutError.new('Each "panel" must define an array :metrics') + raise LayoutError.new('Each "panel" must define an array :metrics') end def for_metrics diff --git a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb new file mode 100644 index 00000000000..2a959854be0 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class EndpointInserter < BaseStage + MissingQueryError = Class.new(DashboardProcessingError) + + def transform! + for_metrics do |metric| + metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) + end + end + + private + + def endpoint_for_metric(metric) + Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( + project, + environment, + proxy_path: query_type(metric), + query: query_for_metric(metric) + ) + end + + def query_type(metric) + metric[:query] ? :query : :query_range + end + + def query_for_metric(metric) + query = metric[query_type(metric)] + + raise MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + + query + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb new file mode 100644 index 00000000000..25e40c70230 --- /dev/null +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'puma/state_file' + +module Gitlab + module Metrics + module Samplers + class PumaSampler < BaseSampler + def metrics + @metrics ||= init_metrics + end + + def init_metrics + { + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), + puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + } + end + + def sample + json_stats = puma_stats + return unless json_stats + + stats = JSON.parse(json_stats) + + if cluster?(stats) + sample_cluster(stats) + else + sample_single_worker(stats) + end + end + + private + + def puma_stats + Puma.stats + rescue NoMethodError + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + nil + end + + def sample_cluster(stats) + set_master_metrics(stats) + + stats['worker_status'].each do |worker| + last_status = worker['last_status'] + labels = { worker: "worker_#{worker['index']}" } + + metrics[:puma_phase].set(labels, worker['phase']) + set_worker_metrics(last_status, labels) if last_status.present? + end + end + + def sample_single_worker(stats) + metrics[:puma_workers].set({}, 1) + metrics[:puma_running_workers].set({}, 1) + + set_worker_metrics(stats) + end + + def cluster?(stats) + stats['worker_status'].present? + end + + def set_master_metrics(stats) + labels = { worker: "master" } + + metrics[:puma_workers].set(labels, stats['workers']) + metrics[:puma_running_workers].set(labels, stats['booted_workers']) + metrics[:puma_stale_workers].set(labels, stats['old_workers']) + metrics[:puma_phase].set(labels, stats['phase']) + end + + def set_worker_metrics(stats, labels = {}) + metrics[:puma_running].set(labels, stats['running']) + metrics[:puma_queued_connections].set(labels, stats['backlog']) + metrics[:puma_active_connections].set(labels, stats['max_threads'] - stats['pool_capacity']) + metrics[:puma_pool_capacity].set(labels, stats['pool_capacity']) + metrics[:puma_max_threads].set(labels, stats['max_threads']) + metrics[:puma_idle_threads].set(labels, stats['running'] + stats['pool_capacity'] - stats['max_threads']) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 18a69321905..17eacbd21d8 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -23,25 +23,32 @@ module Gitlab end def init_metrics - metrics = {} - metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics = { + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + } + GC.stat.keys.each do |key| metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) - metrics end def sample start_time = System.monotonic_time - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - + metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) @@ -61,11 +68,19 @@ module Gitlab metrics[:total_time].increment(labels, GC::Profiler.total_time) end + def set_memory_usage_metrics + memory_usage = System.memory_usage + memory_labels = labels.merge(worker_label) + + metrics[:memory_bytes].set(memory_labels, memory_usage) + metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) + end + def worker_label + return { worker: 'sidekiq' } if Sidekiq.server? return {} unless defined?(Unicorn::Worker) worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no { worker: worker_no } else diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..9af7e0afed4 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,16 +4,16 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler - def initialize(interval) - super(interval) + def metrics + @metrics ||= init_metrics end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) - end - - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') + } end def enabled? @@ -23,14 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('tcp', addr, stats) end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('unix', addr, stats) end + + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +38,13 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def set_unicorn_connection_metrics(type, addr, stats) + labels = { socket_type: type, socket_address: addr } + + metrics[:unicorn_active_connections].set(labels, stats.active) + metrics[:unicorn_queued_connections].set(labels, stats.queued) + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +52,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count + end end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 426496855e3..33c0de91c11 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -23,6 +23,22 @@ module Gitlab def self.file_descriptor_count Dir.glob('/proc/self/fd/*').length end + + def self.max_open_file_descriptors + match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) + + return unless match && match[1] + + match[1].to_i + end + + def self.process_start_time + fields = File.read('/proc/self/stat').split + + # fields[21] is linux proc stat field "(22) starttime". + # The value is expressed in clock ticks, divide by clock ticks for seconds. + ( fields[21].to_i || 0 ) / clk_tck + end else def self.memory_usage 0.0 @@ -31,6 +47,14 @@ module Gitlab def self.file_descriptor_count 0 end + + def self.max_open_file_descriptors + 0 + end + + def self.process_start_time + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -59,6 +83,10 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end + + def self.clk_tck + @clk_tck ||= `getconf CLK_TCK`.to_i + end end end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index e0ac9eec1f2..83204fa5d18 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -36,12 +36,25 @@ module Gitlab hash_arguments = provider['args'].merge(provider_defaults(provider)) # A Hash from the configuration will be passed as is. - provider_arguments << hash_arguments.symbolize_keys + provider_arguments << normalize_hash_arguments(hash_arguments) end provider_arguments end + def normalize_hash_arguments(args) + args.symbolize_keys! + + # Rails 5.1 deprecated the use of string names in the middleware + # (https://github.com/rails/rails/commit/83b767ce), so we need to + # pass in the actual class to Devise. + if args[:strategy_class].is_a?(String) + args[:strategy_class] = args[:strategy_class].constantize + end + + args + end + def provider_defaults(provider) case provider['name'] when 'cas3' @@ -50,6 +63,12 @@ module Gitlab { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } + when 'openid_connect' + # If a name argument is omitted, OmniAuth will expect that the + # matching route is /auth/users/openidconnect instead of + # /auth/users/openid_connect because of + # https://gitlab.com/gitlab-org/gitlab-ce/issues/62208#note_178780341. + { name: 'openid_connect' } else {} end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index aa2c1ac9cef..a07b1246bee 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -58,7 +58,6 @@ module Gitlab uploads users v2 - visual-review-toolbar.js ].freeze # This list should contain all words following `/*namespace_id/:project_id` in diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 16c16aa0265..3a27e26eaba 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -27,15 +27,16 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if Gitlab::SafeRequestStore.store[:peek_enabled] unless data[:cached] - track_query(data[:sql].strip, data[:binds], start, finish) + backtrace = Gitlab::Profiler.clean_backtrace(caller) + track_query(data[:sql].strip, data[:binds], backtrace, start, finish) end end end end - def track_query(raw_query, bindings, start, finish) + def track_query(raw_query, bindings, backtrace, start, finish) duration = (finish - start) * 1000.0 - query_info = { duration: duration.round(3), sql: raw_query } + query_info = { duration: duration.round(3), sql: raw_query, backtrace: backtrace } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb new file mode 100644 index 00000000000..3885a9934d5 --- /dev/null +++ b/lib/gitlab/phabricator_import.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + BaseError = Class.new(StandardError) + + def self.available? + Feature.enabled?(:phabricator_import) && + Gitlab::CurrentSettings.import_sources.include?('phabricator') + end + end +end diff --git a/lib/gitlab/phabricator_import/base_worker.rb b/lib/gitlab/phabricator_import/base_worker.rb new file mode 100644 index 00000000000..b69c65e78f8 --- /dev/null +++ b/lib/gitlab/phabricator_import/base_worker.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +# All workers within a Phabricator import should inherit from this worker and +# implement the `#import` method. The jobs should then be scheduled using the +# `.schedule` class method instead of `.perform_async` +# +# Doing this makes sure that only one job of that type is running at the same time +# for a certain project. This will avoid deadlocks. When a job is already running +# we'll wait for it for 10 times 5 seconds to restart. If the running job hasn't +# finished, by then, we'll retry in 30 seconds. +# +# It also makes sure that we keep the import state of the project up to date: +# - It keeps track of the jobs so we know how many jobs are running for the +# project +# - It refreshes the import jid, so it doesn't get cleaned up by the +# `StuckImportJobsWorker` +# - It marks the import as failed if a job failed to many times +# - It marks the import as finished when all remaining jobs are done +module Gitlab + module PhabricatorImport + class BaseWorker + include ApplicationWorker + include ProjectImportOptions # This marks the project as failed after too many tries + include Gitlab::ExclusiveLeaseHelpers + + class << self + def schedule(project_id, *args) + perform_async(project_id, *args) + add_job(project_id) + end + + def add_job(project_id) + worker_state(project_id).add_job + end + + def remove_job(project_id) + worker_state(project_id).remove_job + end + + def worker_state(project_id) + Gitlab::PhabricatorImport::WorkerState.new(project_id) + end + end + + def perform(project_id, *args) + in_lock("#{self.class.name.underscore}/#{project_id}/#{args}", ttl: 2.hours, sleep_sec: 5.seconds) do + project = Project.find_by_id(project_id) + next unless project + + # Bail if the import job already failed + next unless project.import_state&.in_progress? + + project.import_state.refresh_jid_expiration + + import(project, *args) + + # If this is the last running job, finish the import + project.after_import if self.class.worker_state(project_id).running_count < 2 + + self.class.remove_job(project_id) + end + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + # Reschedule a job if there was already a running one + # Running them at the same time could cause a deadlock updating the same + # resource + self.class.perform_in(30.seconds, project_id, *args) + end + + private + + def import(project, *args) + importer_class.new(project, *args).execute + end + + def importer_class + raise NotImplementedError, "Implement `#{__method__}` on #{self.class}" + end + end + end +end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb new file mode 100644 index 00000000000..fa8b37b20ca --- /dev/null +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Cache + class Map + def initialize(project) + @project = project + end + + def get_gitlab_model(phabricator_id) + cached_info = get(phabricator_id) + return unless cached_info[:classname] && cached_info[:database_id] + + cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + end + + def set_gitlab_model(object, phabricator_id) + set(object.class, object.id, phabricator_id) + end + + private + + attr_reader :project + + def set(klass_name, object_id, phabricator_id) + key = cache_key_for_phabricator_id(phabricator_id) + + redis.with do |r| + r.multi do |multi| + multi.mapped_hmset(key, + { classname: klass_name, database_id: object_id }) + multi.expire(key, timeout) + end + end + end + + def get(phabricator_id) + key = cache_key_for_phabricator_id(phabricator_id) + + redis.with do |r| + r.pipelined do |pipe| + # Extend the TTL when a key was + pipe.expire(key, timeout) + pipe.mapped_hmget(key, :classname, :database_id) + end.last + end + end + + def cache_key_for_phabricator_id(phabricator_id) + "#{Redis::Cache::CACHE_NAMESPACE}/phabricator-import/#{project.id}/#{phabricator_id}" + end + + def redis + Gitlab::Redis::Cache + end + + def timeout + # Setting the timeout to the same one as we do for clearing stuck jobs + # this makes sure all cache is available while the import is running. + StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit.rb b/lib/gitlab/phabricator_import/conduit.rb new file mode 100644 index 00000000000..4c64d737389 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + ApiError = Class.new(Gitlab::PhabricatorImport::BaseError) + ResponseError = Class.new(ApiError) + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb new file mode 100644 index 00000000000..4469a3f5849 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/client.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Client + def initialize(phabricator_url, api_token) + @phabricator_url = phabricator_url + @api_token = api_token + end + + def get(path, params: {}) + response = Gitlab::HTTP.get(build_url(path), body: build_params(params), headers: headers) + Response.parse!(response) + rescue *Gitlab::HTTP::HTTP_ERRORS => e + # Wrap all errors from the API into an API-error. + raise ApiError.new(e) + end + + private + + attr_reader :phabricator_url, :api_token + + def headers + { "Accept" => 'application/json' } + end + + def build_url(path) + URI.join(phabricator_url, '/api/', path).to_s + end + + def build_params(params) + params = params.dup + params.compact! + params.reverse_merge!("api.token" => api_token) + + CGI.unescape(params.to_query) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/maniphest.rb b/lib/gitlab/phabricator_import/conduit/maniphest.rb new file mode 100644 index 00000000000..848b71e49e7 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/maniphest.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Maniphest + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def tasks(after: nil) + TasksResponse.new(get_tasks(after)) + end + + private + + def get_tasks(after) + client.get('maniphest.search', + params: { + after: after, + attachments: { projects: 1, subscribers: 1, columns: 1 } + }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/pagination.rb b/lib/gitlab/phabricator_import/conduit/pagination.rb new file mode 100644 index 00000000000..5f54cccdbc8 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/pagination.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Pagination + def initialize(cursor_json) + @cursor_json = cursor_json + end + + def has_next_page? + next_page.present? + end + + def next_page + cursor_json["after"] + end + + private + + attr_reader :cursor_json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb new file mode 100644 index 00000000000..6053ecfbd5e --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/response.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class Response + def self.parse!(http_response) + unless http_response.success? + raise Gitlab::PhabricatorImport::Conduit::ResponseError, + "Phabricator responded with #{http_response.status}" + end + + response = new(JSON.parse(http_response.body)) + + unless response.success? + raise ResponseError, + "Phabricator Error: #{response.error_code}: #{response.error_info}" + end + + response + rescue JSON::JSONError => e + raise ResponseError.new(e) + end + + def initialize(json) + @json = json + end + + def success? + error_code.nil? + end + + def error_code + json['error_code'] + end + + def error_info + json['error_info'] + end + + def data + json_result&.fetch('data') + end + + def pagination + return unless cursor_info = json_result&.fetch('cursor') + + @pagination ||= Pagination.new(cursor_info) + end + + private + + attr_reader :json + + def json_result + json['result'] + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/tasks_response.rb b/lib/gitlab/phabricator_import/conduit/tasks_response.rb new file mode 100644 index 00000000000..cbcf7259fb2 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/tasks_response.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class TasksResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + delegate :pagination, to: :conduit_response + + def tasks + @tasks ||= conduit_response.data.map do |task_json| + Gitlab::PhabricatorImport::Representation::Task.new(task_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/import_tasks_worker.rb b/lib/gitlab/phabricator_import/import_tasks_worker.rb new file mode 100644 index 00000000000..c36954a8d41 --- /dev/null +++ b/lib/gitlab/phabricator_import/import_tasks_worker.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + class ImportTasksWorker < BaseWorker + def importer_class + Gitlab::PhabricatorImport::Issues::Importer + end + end + end +end diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb new file mode 100644 index 00000000000..c1797f4027e --- /dev/null +++ b/lib/gitlab/phabricator_import/importer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class Importer + def self.async? + true + end + + def self.imports_repository? + # This does not really import a repository, but we want to skip all + # repository related tasks in the `Projects::ImportService` + true + end + + def initialize(project) + @project = project + end + + def execute + Gitlab::Import::SetAsyncJid.set_jid(project) + schedule_first_tasks_page + + true + rescue => e + fail_import(e.message) + + false + end + + private + + attr_reader :project + + def schedule_first_tasks_page + ImportTasksWorker.schedule(project.id) + end + + def fail_import(message) + project.import_state.mark_as_failed(message) + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb new file mode 100644 index 00000000000..a58438452ff --- /dev/null +++ b/lib/gitlab/phabricator_import/issues/importer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Issues + class Importer + def initialize(project, after = nil) + @project, @after = project, after + end + + def execute + schedule_next_batch + + tasks_response.tasks.each do |task| + TaskImporter.new(project, task).execute + end + end + + private + + attr_reader :project, :after + + def schedule_next_batch + return unless tasks_response.pagination.has_next_page? + + Gitlab::PhabricatorImport::ImportTasksWorker + .schedule(project.id, tasks_response.pagination.next_page) + end + + def tasks_response + @tasks_response ||= client.tasks(after: after) + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::Maniphest + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb new file mode 100644 index 00000000000..40d4392cbc1 --- /dev/null +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Issues + class TaskImporter + def initialize(project, task) + @project, @task = project, task + end + + def execute + # TODO: get the user from the project namespace from the username loaded by Phab-id + # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 + issue.author = User.ghost + + # TODO: Reformat the description with attachments, escaping accidental + # links and add attachments + # https://gitlab.com/gitlab-org/gitlab-ce/issues/60603 + issue.assign_attributes(task.issue_attributes) + + save! + + issue + end + + private + + attr_reader :project, :task + + def save! + # Just avoiding an extra redis call, we've already updated the expiry + # when reading the id from the map + was_persisted = issue.persisted? + + issue.save! if issue.changed? + + object_map.set_gitlab_model(issue, task.phabricator_id) unless was_persisted + end + + def issue + @issue ||= find_issue_by_phabricator_id(task.phabricator_id) || + project.issues.new + end + + def find_issue_by_phabricator_id(phabricator_id) + object_map.get_gitlab_model(phabricator_id) + end + + def object_map + Gitlab::PhabricatorImport::Cache::Map.new(project) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb new file mode 100644 index 00000000000..b37a5b44980 --- /dev/null +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class ProjectCreator + def initialize(current_user, params) + @current_user = current_user + @params = params.dup + end + + def execute + return unless import_url.present? && api_token.present? + + project = Projects::CreateService.new(current_user, create_params).execute + return project unless project.persisted? + + project.project_feature.update!(project_feature_attributes) + + project + end + + private + + attr_reader :current_user, :params + + def create_params + { + name: project_name, + path: project_path, + namespace_id: namespace_id, + import_type: 'phabricator', + import_url: Project::UNKNOWN_IMPORT_URL, + import_data: import_data + } + end + + def project_name + params[:name] + end + + def project_path + params[:path] + end + + def namespace_id + params[:namespace_id] || current_user.namespace_id + end + + def import_url + params[:phabricator_server_url] + end + + def api_token + params[:api_token] + end + + def project_feature_attributes + @project_features_attributes ||= begin + # everything disabled except for issues + ProjectFeature::FEATURES.map do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) + end + end + + def import_data + { + data: { + phabricator_url: import_url + }, + credentials: { + api_token: params.fetch(:api_token) + } + } + end + end + end +end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb new file mode 100644 index 00000000000..6aedc71b626 --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Representation + class Task + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def issue_attributes + @issue_attributes ||= { + title: issue_title, + description: issue_description, + state: issue_state, + created_at: issue_created_at, + closed_at: issue_closed_at + } + end + + private + + attr_reader :json + + def issue_title + # The 255 limit is the validation we impose on the Issue title in + # Issuable + @issue_title ||= json['fields']['name'].truncate(255) + end + + def issue_description + json['fields']['description']['raw'] + end + + def issue_state + issue_closed_at.present? ? :closed : :opened + end + + def issue_created_at + return unless json['fields']['dateCreated'] + + @issue_created_at ||= cast_datetime(json['fields']['dateCreated']) + end + + def issue_closed_at + return unless json['fields']['dateClosed'] + + @issue_closed_at ||= cast_datetime(json['fields']['dateClosed']) + end + + def cast_datetime(value) + Time.at(value.to_i) + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb new file mode 100644 index 00000000000..38829e34509 --- /dev/null +++ b/lib/gitlab/phabricator_import/worker_state.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + class WorkerState + def initialize(project_id) + @project_id = project_id + end + + def add_job + redis.with do |r| + r.pipelined do |pipe| + pipe.incr(all_jobs_key) + pipe.expire(all_jobs_key, timeout) + end + end + end + + def remove_job + redis.with do |r| + r.decr(all_jobs_key) + end + end + + def running_count + redis.with { |r| r.get(all_jobs_key) }.to_i + end + + private + + attr_reader :project_id + + def redis + Gitlab::Redis::SharedState + end + + def all_jobs_key + @all_jobs_key ||= "phabricator-import/jobs/project-#{project_id}/job-count" + end + + def timeout + # Make sure we get rid of all the information after a job is marked + # as failed/succeeded + StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 78337518988..0f3b97e2317 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -138,6 +138,12 @@ module Gitlab project end + def filter_milestones_by_project(milestones) + return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) + + milestones.where(project_id: project.id) # rubocop: disable CodeReuse/ActiveRecord + end + def repository_project_ref @repository_project_ref ||= repository_ref || project.default_branch end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 1cd158db2b0..e1579cfddc0 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -102,7 +102,7 @@ module Gitlab @updates[:milestone_id] = nil end - desc _('Copy labels and milestone from other issue or merge request') + desc _('Copy labels and milestone from other issue or merge request in this project') explanation do |source_issuable| _("Copy labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end diff --git a/lib/gitlab/rack_timeout_observer.rb b/lib/gitlab/rack_timeout_observer.rb new file mode 100644 index 00000000000..80d3f7dea60 --- /dev/null +++ b/lib/gitlab/rack_timeout_observer.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + class RackTimeoutObserver + def initialize + @counter = Gitlab::Metrics.counter(:rack_state_total, 'Number of requests in a given rack state') + end + + # returns the Proc to be used as the observer callback block + def callback + method(:log_timeout_exception) + end + + private + + def log_timeout_exception(env) + info = env[::Rack::Timeout::ENV_INFO_KEY] + return unless info + + @counter.increment(labels(info, env)) + end + + def labels(info, env) + params = controller_params(env) || grape_params(env) || {} + + { + controller: params['controller'], + action: params['action'], + route: params['route'], + state: info.state + } + end + + def controller_params(env) + env['action_dispatch.request.parameters'] + end + + def grape_params(env) + endpoint = env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + return unless route + + { 'route' => route } + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 4a097a00101..3daa03d01d6 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -85,6 +85,10 @@ module Gitlab UsersFinder.new(current_user, search: query).execute end + def display_options(_scope) + {} + end + private def projects @@ -103,9 +107,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def milestones - milestones = Milestone.where(project_id: project_ids_relation) - milestones = milestones.search(query) - milestones.reorder('milestones.updated_at DESC') + milestones = Milestone.search(query) + + milestones = filter_milestones_by_project(milestones) + + milestones.reorder('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord @@ -123,6 +129,26 @@ module Gitlab 'projects' end + # Filter milestones by authorized projects. + # For performance reasons project_id is being plucked + # to be used on a smaller query. + # + # rubocop: disable CodeReuse/ActiveRecord + def filter_milestones_by_project(milestones) + project_ids = + milestones.where(project_id: project_ids_relation) + .select(:project_id).distinct + .pluck(:project_id) + + return Milestone.none if project_ids.nil? + + authorized_project_ids_relation = + Project.where(id: project_ids).ids_with_milestone_available_for(current_user) + + milestones.where(project_id: authorized_project_ids_relation) + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 2b7e12639be..0d3e78c0a66 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -44,6 +44,13 @@ module Gitlab config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path + if Rails.env.test? + # Compared to production, tests run in constrained environments. This + # number is meant to grow with the number of concurrent rails requests / + # sidekiq jobs, and concurrency will be low anyway in test. + config[:git] = { catfile_cache_size: 5 } + end + TomlRB.dump(config) end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 641ba70ef83..9a8df719827 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -8,38 +8,68 @@ module Gitlab BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, ports: [], schemes: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false) - return true if url.nil? + # Validates the given url according to the constraints specified by arguments. + # + # ports - Raises error if the given URL port does is not between given ports. + # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is true. + # allow_local_network - Raises error if URL resolves to a link-local address and argument is true. + # ascii_only - Raises error if URL has unicode characters and argument is true. + # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. + # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # + # Returns an array with [<uri>, <original-hostname>]. + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/ParameterLists + def validate!( + url, + ports: [], + schemes: [], + allow_localhost: false, + allow_local_network: true, + ascii_only: false, + enforce_user: false, + enforce_sanitization: false, + dns_rebind_protection: true) + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/ParameterLists + + return [nil, nil] if url.nil? # Param url can be a string, URI or Addressable::URI uri = parse_url(url) validate_html_tags!(uri) if enforce_sanitization - # Allow imports from the GitLab instance itself but only from the configured ports - return true if internal?(uri) - + hostname = uri.hostname port = get_port(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(uri.hostname) - validate_unicode_restriction!(uri) if ascii_only + + unless internal?(uri) + validate_scheme!(uri.scheme, schemes) + validate_port!(port, ports) if ports.any? + validate_user!(uri.user) if enforce_user + validate_hostname!(hostname) + validate_unicode_restriction!(uri) if ascii_only + end begin - addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr| + addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError - return true + return [uri, nil] end + protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + + # Allow url from the GitLab instance itself but only for the configured hostname and ports + return protected_uri_with_hostname if internal?(uri) + validate_localhost!(addrs_info) unless allow_localhost validate_loopback!(addrs_info) unless allow_localhost validate_local_network!(addrs_info) unless allow_local_network validate_link_local!(addrs_info) unless allow_local_network - true + protected_uri_with_hostname end def blocked_url?(*args) @@ -52,6 +82,25 @@ module Gitlab private + # Returns the given URI with IP address as hostname and the original hostname respectively + # in an Array. + # + # It checks whether the resolved IP address matches with the hostname. If not, it changes + # the hostname to the resolved IP address. + # + # The original hostname is used to validate the SSL, given in that scenario + # we'll be making the request to the IP address, instead of using the hostname. + def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + address = addrs_info.first + ip_address = address&.ip_address + + return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname + + uri = uri.dup + uri.hostname = ip_address + [uri, hostname] + end + def get_port(uri) uri.port || uri.default_port end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 880712de5fe..215454fe63c 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -47,6 +47,10 @@ module Gitlab @credentials ||= { user: @url.user.presence, password: @url.password.presence } end + def user + credentials[:user] + end + def full_url @full_url ||= generate_full_url.to_s end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb new file mode 100644 index 00000000000..24d8eac200c --- /dev/null +++ b/lib/quality/test_level.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Quality + class TestLevel + UnknownTestLevelError = Class.new(StandardError) + + TEST_LEVEL_FOLDERS = { + unit: %w[ + bin + config + db + dependencies + factories + finders + frontend + graphql + helpers + initializers + javascripts + lib + migrations + models + policies + presenters + rack_servers + routing + rubocop + serializers + services + sidekiq + tasks + uploaders + validators + views + workers + elastic_integration + ], + integration: %w[ + controllers + mailers + requests + ], + system: ['features'] + }.freeze + + attr_reader :prefix + + def initialize(prefix = nil) + @prefix = prefix + @patterns = {} + @regexps = {} + end + + def pattern(level) + @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb".freeze + end + + def regexp(level) + @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze + end + + def level_for(file_path) + case file_path + when regexp(:unit) + :unit + when regexp(:integration) + :integration + when regexp(:system) + :system + else + raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}." + end + end + end +end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 2235a6ba194..91a52144dd2 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -19,6 +19,7 @@ namespace :gettext do Rake::Task['gettext:po_to_json'].invoke end + desc 'Regenerate gitlab.pot file' task :regenerate do pot_file = 'locale/gitlab.pot' # Remove all translated files, this speeds up finding diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 7a42e4e92a0..a07ae3a418a 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -10,9 +10,15 @@ namespace :gitlab do rake:assets:precompile webpack:compile gitlab:assets:fix_urls + gitlab:assets:compile_vrt ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) end + desc 'GitLab | Assets | Compile visual review toolbar' + task :compile_vrt do + system 'yarn', 'webpack-vrt' + end + desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ee3ef9dad6e..487808a7baa 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -51,9 +51,6 @@ namespace :gitlab do end end - # (Re)create hooks - Rake::Task['gitlab:shell:create_hooks'].invoke - Gitlab::Shell.ensure_secret_token! end @@ -78,15 +75,6 @@ namespace :gitlab do end end end - - desc 'Create or repair repository hooks symlink' - task create_hooks: :gitlab_environment do - warn_user_is_not_gitlab - - puts 'Creating/Repairing hooks symlinks for all repositories' - system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) - puts 'done'.color(:green) - end end def setup diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 2eddcb3c777..c881ad4cf12 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -1,7 +1,32 @@ +# frozen_string_literal: true + +return if Rails.env.production? + Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') namespace :spec do - desc 'GitLab | Rspec | Run request specs' + desc 'GitLab | RSpec | Run unit tests' + RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:unit) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run integration tests' + RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:integration) + t.rspec_opts = args[:rspec_opts] + end + + desc 'GitLab | RSpec | Run system tests' + RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args| + require_dependency 'quality/test_level' + t.pattern = Quality::TestLevel.new.pattern(:system) + t.rspec_opts = args[:rspec_opts] + end + + desc '[Deprecated] Use the "bin/rspec --tag api" instead' task :api do cmds = [ %w(rake gitlab:setup), @@ -10,7 +35,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run feature specs' + desc '[Deprecated] Use the "spec:system" task instead' task :feature do cmds = [ %w(rake gitlab:setup), @@ -19,7 +44,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run model specs' + desc '[Deprecated] Use "bin/rspec spec/models" instead' task :models do cmds = [ %w(rake gitlab:setup), @@ -28,7 +53,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run service specs' + desc '[Deprecated] Use "bin/rspec spec/services" instead' task :services do cmds = [ %w(rake gitlab:setup), @@ -37,7 +62,7 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run lib specs' + desc '[Deprecated] Use "bin/rspec spec/lib" instead' task :lib do cmds = [ %w(rake gitlab:setup), @@ -45,15 +70,6 @@ namespace :spec do ] run_commands(cmds) end - - desc 'GitLab | Rspec | Run other specs' - task :other do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) - ] - run_commands(cmds) - end end desc "GitLab | Run specs" |