diff options
Diffstat (limited to 'lib')
423 files changed, 6608 insertions, 3886 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 54e5cc5c8d0..2a3033753f7 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -52,8 +52,6 @@ module API api_endpoint = env['api.endpoint'] feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s - header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category - Gitlab::ApplicationContext.push( user: -> { @current_user }, project: -> { @project }, @@ -170,11 +168,11 @@ module API mount ::API::ErrorTracking mount ::API::Events mount ::API::FeatureFlags - mount ::API::FeatureFlagScopes mount ::API::FeatureFlagsUserLists mount ::API::Features mount ::API::Files mount ::API::FreezePeriods + mount ::API::GroupAvatar mount ::API::GroupBoards mount ::API::GroupClusters mount ::API::GroupExport @@ -224,10 +222,12 @@ module API mount ::API::NpmInstancePackages mount ::API::GenericPackages mount ::API::GoProxy + mount ::API::HelmPackages mount ::API::Pages mount ::API::PagesDomains mount ::API::ProjectClusters mount ::API::ProjectContainerRepositories + mount ::API::ProjectDebianDistributions mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 1ee120f982a..0db5bb82296 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -17,6 +17,10 @@ module API authorize! :download_code, user_project end + rescue_from Gitlab::Git::Repository::NoRepository do + not_found! + end + helpers do params :filter_params do optional :search, type: String, desc: 'Return list of branches matching the search criteria' diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 33980b38e2b..c4e0b699524 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -98,6 +98,9 @@ module API optional :architecture, type: String, desc: %q(Runner's architecture) optional :executor, type: String, desc: %q(Runner's executor) optional :features, type: Hash, desc: %q(Runner's features) + optional :config, type: Hash, desc: %q(Runner's config) do + optional :gpus, type: String, desc: %q(GPUs enabled) + end end optional :session, type: Hash, desc: %q(Runner's session data) do optional :url, type: String, desc: %q(Session's url) @@ -165,7 +168,6 @@ module API params do requires :token, type: String, desc: %q(Runners's authentication token) requires :id, type: Integer, desc: %q(Job's ID) - optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum) optional :failure_reason, type: String, desc: %q(Job's failure_reason) diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index e199111c975..27fee7fdea2 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -96,10 +96,8 @@ module API protected: user_project.protected_for?(ref) ) - optional_attributes = - attributes_for_keys(%w[target_url description coverage]) - - status.update(optional_attributes) if optional_attributes.any? + updatable_optional_attributes = %w[target_url description coverage] + status.assign_attributes(attributes_for_keys(updatable_optional_attributes)) if status.valid? status.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, user_project, default_enabled: :yaml) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index bd9f83ac24c..541a37b0abe 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - require 'mime/types' module API @@ -41,6 +40,7 @@ module API optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response' optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges' optional :order, type: String, desc: 'List commits in order', default: 'default', values: %w[default topo] + optional :trailers, type: Boolean, desc: 'Parse and include Git trailers for every commit', default: false use :pagination end get ':id/repository/commits' do @@ -62,7 +62,8 @@ module API after: after, all: all, first_parent: first_parent, - order: order) + order: order, + trailers: params[:trailers]) serializer = with_stats ? Entities::CommitWithStats : Entities::Commit @@ -203,6 +204,7 @@ module API requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' requires :branch, type: String, desc: 'The name of the branch', allow_blank: false optional :dry_run, type: Boolean, default: false, desc: "Does not commit any changes" + optional :message, type: String, desc: 'A custom commit message to use for the picked commit' end post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize_push_to_branch!(params[:branch]) @@ -216,7 +218,8 @@ module API commit: commit, start_branch: params[:branch], branch_name: params[:branch], - dry_run: params[:dry_run] + dry_run: params[:dry_run], + message: params[:message] } result = ::Commits::CherryPickService diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 115a6b8ac4f..7b3750b37ee 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -137,7 +137,7 @@ module API bad_request! end - track_package_event('push_package', :composer) + track_package_event('push_package', :composer, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) ::Packages::Composer::CreatePackageService .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) @@ -161,7 +161,7 @@ module API not_found! unless metadata - track_package_event('pull_package', :composer) + track_package_event('pull_package', :composer, project: unauthorized_user_project, namespace: unauthorized_user_project.namespace) send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index eb762be8285..3194cdebde8 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -255,7 +255,7 @@ module API delete do authorize!(:destroy_package, project) - track_package_event('delete_package', :conan, category: 'API::ConanPackages') + track_package_event('delete_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) package.destroy end diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb new file mode 100644 index 00000000000..4670c3e3521 --- /dev/null +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +module API + module Concerns + module Packages + module DebianDistributionEndpoints + extend ActiveSupport::Concern + + included do + include PaginationParams + + feature_category :package_registry + + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication + + namespace 'debian_distributions' do + helpers do + params :optional_distribution_params do + optional :suite, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Suite' + optional :origin, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Origin' + optional :label, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Label' + optional :version, type: String, regexp: Gitlab::Regex.debian_version_regex, desc: 'The Debian Version' + optional :description, type: String, desc: 'The Debian Description' + optional :valid_time_duration_seconds, type: Integer, desc: 'The duration before the Release file should be considered expired by the client' + + optional :components, type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + regexp: Gitlab::Regex.debian_component_regex, + desc: 'The list of Components' + optional :architectures, type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + regexp: Gitlab::Regex.debian_architecture_regex, + desc: 'The list of Architectures' + end + end + + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end + + content_type :json, 'application/json' + format :json + + # POST {projects|groups}/:id/debian_distributions + desc 'Create a Debian Distribution' do + detail 'This feature was introduced in 14.0' + success ::API::Entities::Packages::Debian::Distribution + end + + params do + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + use :optional_distribution_params + end + post '/' do + authorize_create_package!(project_or_group) + + distribution_params = declared_params(include_missing: false) + result = ::Packages::Debian::CreateDistributionService.new(project_or_group, current_user, distribution_params).execute + distribution = result.payload[:distribution] + + if result.success? + present distribution, with: ::API::Entities::Packages::Debian::Distribution + else + render_validation_error!(distribution) + end + end + + # GET {projects|groups}/:id/debian_distributions + desc 'Get a list of Debian Distributions' do + detail 'This feature was introduced in 14.0' + success ::API::Entities::Packages::Debian::Distribution + end + + params do + use :pagination + optional :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + use :optional_distribution_params + end + get '/' do + distribution_params = declared_params(include_missing: false) + distributions = ::Packages::Debian::DistributionsFinder.new(project_or_group, distribution_params).execute + + present paginate(distributions), with: ::API::Entities::Packages::Debian::Distribution + end + + # GET {projects|groups}/:id/debian_distributions/:codename + desc 'Get a Debian Distribution' do + detail 'This feature was introduced in 14.0' + success ::API::Entities::Packages::Debian::Distribution + end + + params do + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + end + get '/:codename' do + distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! + + present distribution, with: ::API::Entities::Packages::Debian::Distribution + end + + # PUT {projects|groups}/:id/debian_distributions/:codename + desc 'Update a Debian Distribution' do + detail 'This feature was introduced in 14.0' + success ::API::Entities::Packages::Debian::Distribution + end + + params do + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + use :optional_distribution_params + end + put '/:codename' do + authorize_create_package!(project_or_group) + + distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! + distribution_params = declared_params(include_missing: false).except(:codename) + result = ::Packages::Debian::UpdateDistributionService.new(distribution, distribution_params).execute + distribution = result.payload[:distribution] + + if result.success? + present distribution, with: ::API::Entities::Packages::Debian::Distribution + else + render_validation_error!(distribution) + end + end + + # DELETE {projects|groups}/:id/debian_distributions/:codename + desc 'Delete a Debian Distribution' do + detail 'This feature was introduced in 14.0' + end + + params do + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + use :optional_distribution_params + end + delete '/:codename' do + authorize_destroy_package!(project_or_group) + + distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! + + accepted! if distribution.destroy + + render_api_error!('Failed to delete distribution', 400) + end + end + end + end + end + end +end diff --git a/lib/api/concerns/packages/debian_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 6fc7c439464..c79ae3068b4 100644 --- a/lib/api/concerns/packages/debian_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -3,7 +3,7 @@ module API module Concerns module Packages - module DebianEndpoints + module DebianPackageEndpoints extend ActiveSupport::Concern DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze @@ -32,23 +32,17 @@ module API helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication - format :txt - content_type :txt, 'text/plain' - - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end + namespace 'packages/debian' do + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end - before do - require_packages_enabled! - end + format :txt + content_type :txt, 'text/plain' - namespace 'packages/debian' do params do requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex end @@ -59,7 +53,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + route_setting :authentication, authenticate_non_public: true get 'Release.gpg' do not_found! end @@ -69,7 +63,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + route_setting :authentication, authenticate_non_public: true get 'Release' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Release' @@ -80,7 +74,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + route_setting :authentication, authenticate_non_public: true get 'InRelease' do not_found! end @@ -96,7 +90,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + route_setting :authentication, authenticate_non_public: true get 'Packages' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Packages' @@ -119,7 +113,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + route_setting :authentication, authenticate_non_public: true get ':file_name', requirements: FILE_NAME_REQUIREMENTS do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO File' diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 5364eeb1880..208daeb3037 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -58,7 +58,8 @@ module API end get 'index', format: :json do authorize_read_package!(project_or_group) - track_package_event('cli_metadata', :nuget, category: 'API::NugetPackages') + + track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group), with: ::API::Entities::Nuget::ServiceIndex @@ -117,7 +118,7 @@ module API results = search_packages(params[:q], search_options) - track_package_event('search_package', :nuget, category: 'API::NugetPackages') + track_package_event('search_package', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) present ::Packages::Nuget::SearchResultsPresenter.new(results), with: ::API::Entities::Nuget::SearchResults diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 06edab662bf..c6116a8b28f 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -7,6 +7,14 @@ module API end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + before do require_packages_enabled! @@ -16,7 +24,7 @@ module API end namespace ':id/-' do - include ::API::Concerns::Packages::DebianEndpoints + include ::API::Concerns::Packages::DebianPackageEndpoints end end end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 0ed828fd639..70ddf9dea37 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -7,7 +7,15 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + after_validation do require_packages_enabled! not_found! unless ::Feature.enabled?(:debian_packages, user_project) @@ -16,13 +24,20 @@ module API end namespace ':id' do - include ::API::Concerns::Packages::DebianEndpoints + helpers do + def project_or_group + user_project + end + end + + include ::API::Concerns::Packages::DebianPackageEndpoints params do requires :file_name, type: String, desc: 'The file name' end namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do + format :txt content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE # PUT {projects|groups}/:id/packages/debian/:file_name @@ -35,8 +50,22 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) - track_package_event('push_package', :debian) + file_params = { + file: params['file'], + file_name: params['file_name'], + file_sha1: params['file.sha1'], + file_md5: params['file.md5'] + } + + package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute + + package_file = ::Packages::Debian::CreatePackageFileService.new(package, file_params).execute + + if params['file_name'].end_with? '.changes' + ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker + end + track_package_event('push_package', :debian, user: current_user, project: authorized_user_project, namespace: authorized_user_project.namespace) created! rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index 2de49d6ed40..c75b74b4368 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -4,15 +4,13 @@ module API module Entities class BasicProjectDetails < Entities::ProjectIdentity include ::API::ProjectsRelationBuilder + include Gitlab::Utils::StrongMemoize expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 - expose :tag_list do |project| - # Tags is a preloaded association. If we perform then sorting - # through the database, it will trigger a new query, ending up - # in an N+1 if we have several projects - project.tags.pluck(:name).sort # rubocop:disable CodeReuse/ActiveRecord - end + + expose :topic_names, as: :tag_list + expose :topic_names, as: :topics expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url @@ -40,16 +38,29 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) - # Preloading tags, should be done with using only `:tags`, - # as `:tags` are defined as: `has_many :tags, through: :taggings` - # N+1 is solved then by using `subject.tags.map(&:name)` + # Preloading topics, should be done with using only `:topics`, + # as `:topics` are defined as: `has_many :topics, through: :taggings` + # N+1 is solved then by using `subject.topics.map(&:name)` # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555 projects_relation.preload(:project_feature, :route) - .preload(:import_state, :tags) + .preload(:import_state, :topics) .preload(:auto_devops) .preload(namespace: [:route, :owner]) end # rubocop: enable CodeReuse/ActiveRecord + + private + + alias_method :project, :object + + def topic_names + # Topics is a preloaded association. If we perform then sorting + # through the database, it will trigger a new query, ending up + # in an N+1 if we have several projects + strong_memoize(:topic_names) do + project.topics.pluck(:name).sort # rubocop:disable CodeReuse/ActiveRecord + end + end end end end diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb index 3eaf896f1ac..fd23c23b980 100644 --- a/lib/api/entities/commit.rb +++ b/lib/api/entities/commit.rb @@ -9,6 +9,7 @@ module API expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date + expose :trailers expose :web_url do |commit, _options| Gitlab::UrlBuilder.build(commit) diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb index e63a3fc1334..408254a89be 100644 --- a/lib/api/entities/group_detail.rb +++ b/lib/api/entities/group_detail.rb @@ -29,11 +29,7 @@ module API end def projects_limit - if ::Feature.enabled?(:limit_projects_in_groups_api, default_enabled: true) - GroupProjectsFinder::DEFAULT_PROJECTS_LIMIT - else - nil - end + GroupProjectsFinder::DEFAULT_PROJECTS_LIMIT end end end diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index d27cc5498bd..6c332870228 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -23,7 +23,7 @@ module API expose :issue_type, as: :type, format_with: :upcase, - documentation: { type: "String", desc: "One of #{Issue.issue_types.keys.map(&:upcase)}" } + documentation: { type: "String", desc: "One of #{::Issue.issue_types.keys.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first diff --git a/lib/api/entities/label_basic.rb b/lib/api/entities/label_basic.rb index 00ecea26ec3..ed52688638e 100644 --- a/lib/api/entities/label_basic.rb +++ b/lib/api/entities/label_basic.rb @@ -3,7 +3,7 @@ module API module Entities class LabelBasic < Grape::Entity - expose :id, :name, :color, :description, :description_html, :text_color, :remove_on_close + expose :id, :name, :color, :description, :description_html, :text_color end end end diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index cf8d03bf176..d5cf2f653db 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -36,7 +36,11 @@ module API merge_request.labels.map(&:title).sort end end - expose :work_in_progress?, as: :work_in_progress + expose :draft?, as: :draft + + # [Deprecated] see draft + # + expose :draft?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index 2f60a0bf6bd..1efd457aa5f 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -25,8 +25,12 @@ module API expose :status expose :_links do - expose :web_path do |package| - ::Gitlab::Routing.url_helpers.project_package_path(package.project, package) + expose :web_path do |package, opts| + if package.infrastructure_package? + ::Gitlab::Routing.url_helpers.namespace_project_infrastructure_registry_path(opts[:namespace], package.project, package) + else + ::Gitlab::Routing.url_helpers.project_package_path(package.project, package) + end end expose :delete_api_path, if: can_destroy(:package, &:project) do |package| diff --git a/lib/api/entities/packages/debian/distribution.rb b/lib/api/entities/packages/debian/distribution.rb new file mode 100644 index 00000000000..97a3c479f40 --- /dev/null +++ b/lib/api/entities/packages/debian/distribution.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + module Entities + module Packages + module Debian + class Distribution < Grape::Entity + expose :id + expose :codename + expose :suite + expose :origin + expose :label + expose :version + expose :description + expose :valid_time_duration_seconds + + expose :component_names, as: :components + expose :architecture_names, as: :architectures + end + end + end + end +end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 442013c07dd..68d91fc6970 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -43,7 +43,6 @@ module API expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :resolve_outdated_diff_discussions - expose :container_registry_enabled expose :container_expiration_policy, using: Entities::ContainerExpirationPolicy, if: -> (project, _) { project.container_expiration_policy } @@ -54,6 +53,13 @@ module API expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } + expose(:container_registry_enabled) do |project, options| + if ::Feature.enabled?(:read_container_registry_access_level, project.namespace, default_enabled: :yaml) + project.feature_available?(:container_registry, options[:current_user]) + else + project.read_attribute(:container_registry_enabled) + end + end expose :service_desk_enabled expose :service_desk_address @@ -89,6 +95,7 @@ module API expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :ci_default_git_depth expose :ci_forward_deployment_enabled + expose :ci_job_token_scope_enabled expose :public_builds, as: :public_jobs expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| project.build_allow_git_fetch ? 'fetch' : 'clone' @@ -108,6 +115,7 @@ module API expose :remove_source_branch_after_merge expose :printing_merge_request_link_enabled expose :merge_method + expose :squash_option expose :suggestion_commit_message expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) @@ -120,12 +128,13 @@ module API expose :repository_storage, if: ->(project, options) { Ability.allowed?(options[:current_user], :change_repository_storage, project) } + expose :keep_latest_artifacts_available?, as: :keep_latest_artifact # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) - # Preloading tags, should be done with using only `:tags`, - # as `:tags` are defined as: `has_many :tags, through: :taggings` - # N+1 is solved then by using `subject.tags.map(&:name)` + # Preloading topics, should be done with using only `:topics`, + # as `:topics` are defined as: `has_many :topics, through: :taggings` + # N+1 is solved then by using `subject.topics.map(&:name)` # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555 super(projects_relation).preload(group: :namespace_settings) .preload(:ci_cd_settings) @@ -136,7 +145,7 @@ module API .preload(project_group_links: { group: :route }, fork_network: :root_project, fork_network_member: :forked_from_project, - forked_from_project: [:route, :forks, :tags, :group, :project_feature, namespace: [:route, :owner]]) + forked_from_project: [:route, :topics, :group, :project_feature, namespace: [:route, :owner]]) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities/project_repository_storage.rb b/lib/api/entities/project_repository_storage.rb new file mode 100644 index 00000000000..0816bebde2c --- /dev/null +++ b/lib/api/entities/project_repository_storage.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class ProjectRepositoryStorage < Grape::Entity + include Gitlab::Routing + + expose :disk_path do |project| + project.repository.disk_path + end + + expose :id, as: :project_id + expose :repository_storage, :created_at + end + end +end diff --git a/lib/api/entities/runner.rb b/lib/api/entities/runner.rb index 6165b54cddb..e78f14cf920 100644 --- a/lib/api/entities/runner.rb +++ b/lib/api/entities/runner.rb @@ -8,6 +8,7 @@ module API expose :ip_address expose :active expose :instance_type?, as: :is_shared + expose :runner_type expose :name expose :online?, as: :online expose :status diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb index f05e593a302..af885aaf0eb 100644 --- a/lib/api/entities/snippet.rb +++ b/lib/api/entities/snippet.rb @@ -5,16 +5,22 @@ module API class Snippet < BasicSnippet expose :author, using: Entities::UserBasic expose :file_name do |snippet| - snippet.file_name_on_repo || snippet.file_name + snippet_files.first || snippet.file_name end expose :files do |snippet, options| - snippet.list_files.map do |file| + snippet_files.map do |file| { path: file, raw_url: Gitlab::UrlBuilder.build(snippet, file: file, ref: snippet.repository.root_ref) } end end + + private + + def snippet_files + @snippet_files ||= object.list_files + end end end end diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb index 7a6df9b6c59..ceee6c610d3 100644 --- a/lib/api/entities/user_preferences.rb +++ b/lib/api/entities/user_preferences.rb @@ -3,7 +3,7 @@ module API module Entities class UserPreferences < Grape::Entity - expose :id, :user_id, :view_diffs_file_by_file + expose :id, :user_id, :view_diffs_file_by_file, :show_whitespace_in_diffs end end end diff --git a/lib/api/feature_flag_scopes.rb b/lib/api/feature_flag_scopes.rb deleted file mode 100644 index 3f3bf4d9f42..00000000000 --- a/lib/api/feature_flag_scopes.rb +++ /dev/null @@ -1,160 +0,0 @@ -# frozen_string_literal: true - -module API - class FeatureFlagScopes < ::API::Base - include PaginationParams - - ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS = FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS - .merge(environment_scope: API::NO_SLASH_URL_PART_REGEX) - - feature_category :feature_flags - - before do - authorize_read_feature_flags! - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - resource :feature_flag_scopes do - desc 'Get all effective feature flags under the environment' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::DetailedLegacyScope - end - params do - requires :environment, type: String, desc: 'The environment name' - end - get do - present scopes_for_environment, with: ::API::Entities::FeatureFlag::DetailedLegacyScope - end - end - - params do - requires :name, type: String, desc: 'The name of the feature flag' - end - resource 'feature_flags/:name', requirements: FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS do - resource :scopes do - desc 'Get all scopes of a feature flag' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::LegacyScope - end - params do - use :pagination - end - get do - present paginate(feature_flag.scopes), with: ::API::Entities::FeatureFlag::LegacyScope - end - - desc 'Create a scope of a feature flag' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::LegacyScope - end - params do - requires :environment_scope, type: String, desc: 'The environment scope of the scope' - requires :active, type: Boolean, desc: 'Whether the scope is active' - requires :strategies, type: JSON, desc: 'The strategies of the scope' - end - post do - authorize_update_feature_flag! - - result = ::FeatureFlags::UpdateService - .new(user_project, current_user, scopes_attributes: [declared_params]) - .execute(feature_flag) - - if result[:status] == :success - present scope, with: ::API::Entities::FeatureFlag::LegacyScope - else - render_api_error!(result[:message], result[:http_status]) - end - end - - params do - requires :environment_scope, type: String, desc: 'URL-encoded environment scope' - end - resource ':environment_scope', requirements: ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS do - desc 'Get a scope of a feature flag' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::LegacyScope - end - get do - present scope, with: ::API::Entities::FeatureFlag::LegacyScope - end - - desc 'Update a scope of a feature flag' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::LegacyScope - end - params do - optional :active, type: Boolean, desc: 'Whether the scope is active' - optional :strategies, type: JSON, desc: 'The strategies of the scope' - end - put do - authorize_update_feature_flag! - - scope_attributes = declared_params.merge(id: scope.id) - - result = ::FeatureFlags::UpdateService - .new(user_project, current_user, scopes_attributes: [scope_attributes]) - .execute(feature_flag) - - if result[:status] == :success - updated_scope = result[:feature_flag].scopes - .find { |scope| scope.environment_scope == params[:environment_scope] } - - present updated_scope, with: ::API::Entities::FeatureFlag::LegacyScope - else - render_api_error!(result[:message], result[:http_status]) - end - end - - desc 'Delete a scope from a feature flag' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag::LegacyScope - end - delete do - authorize_update_feature_flag! - - param = { scopes_attributes: [{ id: scope.id, _destroy: true }] } - - result = ::FeatureFlags::UpdateService - .new(user_project, current_user, param) - .execute(feature_flag) - - if result[:status] == :success - status :no_content - else - render_api_error!(result[:message], result[:http_status]) - end - end - end - end - end - end - - helpers do - def authorize_read_feature_flags! - authorize! :read_feature_flag, user_project - end - - def authorize_update_feature_flag! - authorize! :update_feature_flag, feature_flag - end - - def feature_flag - @feature_flag ||= user_project.operations_feature_flags - .find_by_name!(params[:name]) - end - - def scope - @scope ||= feature_flag.scopes - .find_by_environment_scope!(CGI.unescape(params[:environment_scope])) - end - - def scopes_for_environment - Operations::FeatureFlagScope - .for_unleash_client(user_project, params[:environment]) - end - end - end -end diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index 6fdc4535be3..fb5858bc10b 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -90,56 +90,11 @@ module API end get do authorize_read_feature_flag! + exclude_legacy_flags_check! present_entity(feature_flag) end - desc 'Enable a strategy for a feature flag on an environment' do - detail 'This feature was introduced in GitLab 12.5' - success ::API::Entities::FeatureFlag - end - params do - requires :environment_scope, type: String, desc: 'The environment scope of the feature flag' - requires :strategy, type: JSON, desc: 'The strategy to be enabled on the scope' - end - post :enable do - not_found! unless Feature.enabled?(:feature_flag_api, user_project) - render_api_error!('Version 2 flags not supported', :unprocessable_entity) if new_version_flag_present? - - result = ::FeatureFlags::EnableService - .new(user_project, current_user, params).execute - - if result[:status] == :success - status :ok - present_entity(result[:feature_flag]) - else - render_api_error!(result[:message], result[:http_status]) - end - end - - desc 'Disable a strategy for a feature flag on an environment' do - detail 'This feature is going to be introduced in GitLab 12.5 if `feature_flag_api` feature flag is removed' - success ::API::Entities::FeatureFlag - end - params do - requires :environment_scope, type: String, desc: 'The environment scope of the feature flag' - requires :strategy, type: JSON, desc: 'The strategy to be disabled on the scope' - end - post :disable do - not_found! unless Feature.enabled?(:feature_flag_api, user_project) - render_api_error!('Version 2 flags not supported', :unprocessable_entity) if feature_flag.new_version_flag? - - result = ::FeatureFlags::DisableService - .new(user_project, current_user, params).execute - - if result[:status] == :success - status :ok - present_entity(result[:feature_flag]) - else - render_api_error!(result[:message], result[:http_status]) - end - end - desc 'Update a feature flag' do detail 'This feature was introduced in GitLab 13.2' success ::API::Entities::FeatureFlag @@ -162,6 +117,7 @@ module API end put do authorize_update_feature_flag! + exclude_legacy_flags_check! render_api_error!('PUT operations are not supported for legacy feature flags', :unprocessable_entity) if feature_flag.legacy_flag? attrs = declared_params(include_missing: false) @@ -232,6 +188,10 @@ module API @feature_flag ||= user_project.operations_feature_flags.find_by_name!(params[:feature_flag_name]) end + def project + @project ||= feature_flag.project + end + def new_version_flag_present? user_project.operations_feature_flags.new_version_flag.find_by_name(params[:name]).present? end @@ -245,6 +205,12 @@ module API hash[key] = yield(hash[key]) if hash.key?(key) hash end + + def exclude_legacy_flags_check! + if feature_flag.legacy_flag? + not_found! + end + end end end end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index d0680ad7bc5..a57d6bbcd2a 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -62,7 +62,7 @@ module API authorize_upload!(project) bad_request!('File is too large') if max_file_size_exceeded? - ::Gitlab::Tracking.event(self.options[:for].name, 'push_package') + ::Gitlab::Tracking.event(self.options[:for].name, 'push_package', user: current_user, project: project, namespace: project.namespace) create_package_file_params = declared_params.merge(build: current_authenticated_job) ::Packages::Generic::CreatePackageFileService @@ -96,7 +96,7 @@ module API package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! - ::Gitlab::Tracking.event(self.options[:for].name, 'pull_package') + ::Gitlab::Tracking.event(self.options[:for].name, 'pull_package', user: current_user, project: project, namespace: project.namespace) present_carrierwave_file!(package_file.file) end diff --git a/lib/api/group_avatar.rb b/lib/api/group_avatar.rb new file mode 100644 index 00000000000..ddf6787f913 --- /dev/null +++ b/lib/api/group_avatar.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + class GroupAvatar < ::API::Base + helpers Helpers::GroupsHelpers + + feature_category :subgroups + + resource :groups do + desc 'Download the group avatar' do + detail 'This feature was introduced in GitLab 14.0' + end + params do + requires :id, type: String, desc: 'The group id' + end + get ':id/avatar' do + present_carrierwave_file!(user_group.avatar) + end + end + end +end diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 4fede0ad583..96175f31696 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -31,7 +31,7 @@ module API user: current_user, subject: user_group ).execute - track_package_event('list_repositories', :container) + track_package_event('list_repositories', :container, user: current_user, namespace: user_group) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 6134515032f..7e4fdba6033 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -23,7 +23,11 @@ module API check_rate_limit! :group_download_export, [current_user, user_group] if user_group.export_file_exists? - present_carrierwave_file!(user_group.export_file) + if user_group.export_archive_exists? + present_carrierwave_file!(user_group.export_file) + else + render_api_error!('The group export file is not available yet', 404) + end else render_api_error!('404 Not found or has expired', 404) end diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index ab4e91ff925..d9010dfd329 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -43,7 +43,7 @@ module API declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless, :status) ).execute - present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true + present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true, namespace: user_group.root_ancestor end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 1a604e70bf1..0efb8b57885 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -372,7 +372,7 @@ module API expires_at: params[:expires_at] } - result = ::Groups::GroupLinks::CreateService.new(shared_with_group, current_user, group_link_create_params).execute(shared_group) + result = ::Groups::GroupLinks::CreateService.new(shared_group, shared_with_group, current_user, group_link_create_params).execute shared_group.preload_shared_group_links if result[:status] == :success diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb new file mode 100644 index 00000000000..dc5630a1395 --- /dev/null +++ b/lib/api/helm_packages.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +### +# API endpoints for the Helm package registry +module API + class HelmPackages < ::API::Base + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication + + feature_category :package_registry + + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + content_type :binary, 'application/octet-stream' + + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end + + before do + require_packages_enabled! + end + + after_validation do + not_found! unless Feature.enabled?(:helm_packages, authorized_user_project) + end + + params do + requires :id, type: String, desc: 'The ID or full path of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/helm' do + desc 'Download a chart' do + detail 'This feature was introduced in GitLab 14.0' + end + params do + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + requires :file_name, type: String, desc: 'Helm package file name' + end + get ":channel/charts/:file_name.tgz", requirements: FILE_NAME_REQUIREMENTS do + authorize_read_package!(authorized_user_project) + + package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").execute.last! + + track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) + + present_carrierwave_file!(package_file.file) + end + end + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 632717e1b73..6ce04be373f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -74,6 +74,11 @@ module API save_current_user_in_env(@current_user) if @current_user + if @current_user + ::Gitlab::Database::LoadBalancing::RackMiddleware + .stick_or_unstick(env, :user, @current_user.id) + end + @current_user end # rubocop:enable Gitlab/ModuleWithInstanceVariables @@ -482,9 +487,8 @@ module API def handle_api_exception(exception) if report_exception?(exception) define_params_for_grape_middleware - Gitlab::ApplicationContext.with_context(user: current_user) do - Gitlab::ErrorTracking.track_exception(exception) - end + Gitlab::ApplicationContext.push(user: current_user) + Gitlab::ErrorTracking.track_exception(exception) end # This is used with GrapeLogging::Loggers::ExceptionLogger @@ -599,6 +603,7 @@ module API :custom_attributes, :last_activity_after, :last_activity_before, + :topic, :repository_storage) .symbolize_keys .compact @@ -611,7 +616,6 @@ module API finder_params[:user] = params.delete(:user) if params[:user] finder_params[:id_after] = sanitize_id_param(params[:id_after]) if params[:id_after] finder_params[:id_before] = sanitize_id_param(params[:id_before]) if params[:id_before] - finder_params[:tag] = params[:topic] if params[:topic].present? finder_params end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index 796b8928243..da0ee8f207e 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -5,34 +5,27 @@ module API module LabelHelpers extend Grape::API::Helpers - params :optional_label_params do - optional :description, type: String, desc: 'The description of the label' - optional :remove_on_close, type: Boolean, desc: 'Whether the label should be removed from an issue when the issue is closed' - end - params :label_create_params do requires :name, type: String, desc: 'The name of the label to be created' requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - - use :optional_label_params + optional :description, type: String, desc: 'The description of label to be created' end params :label_update_params do optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - - use :optional_label_params + optional :description, type: String, desc: 'The new description of label' end params :project_label_update_params do use :label_update_params optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true - at_least_one_of :new_name, :color, :description, :priority, :remove_on_close + at_least_one_of :new_name, :color, :description, :priority end params :group_label_update_params do use :label_update_params - at_least_one_of :new_name, :color, :description, :remove_on_close + at_least_one_of :new_name, :color, :description end def find_label(parent, id_or_title, params = { include_ancestor_groups: true }) diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb index c32ce199dd6..6c381d85cd8 100644 --- a/lib/api/helpers/packages/basic_auth_helpers.rb +++ b/lib/api/helpers/packages/basic_auth_helpers.rb @@ -22,6 +22,14 @@ module API unauthorized_user_project || not_found! end + def unauthorized_user_group + @unauthorized_user_group ||= find_group(params[:id]) + end + + def unauthorized_user_group! + unauthorized_user_group || not_found! + end + def authorized_user_project @authorized_user_project ||= authorized_project_find! end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index b18f52b5be6..4b6dac39348 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -155,7 +155,7 @@ module API conan_package_reference: params[:conan_package_reference] ).execute! - track_package_event('pull_package', :conan, category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY + track_package_event('pull_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY present_carrierwave_file!(package_file.file) end @@ -170,7 +170,7 @@ module API def track_push_package_event if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate - track_package_event('push_package', :conan, category: 'API::ConanPackages') + track_package_event('push_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index d9c0b4f67c8..69a83043617 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -16,6 +16,7 @@ module API optional :build_coverage_regex, type: String, desc: 'Test coverage parsing' optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :service_desk_enabled, type: Boolean, desc: 'Disable or enable the service desk' + optional :keep_latest_artifact, type: Boolean, desc: 'Indicates if the latest artifact should be kept for this project.' # TODO: remove in API v5, replaced by *_access_level optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' @@ -51,7 +52,8 @@ module API optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :allow_merge_on_skipped_pipeline, type: Boolean, desc: 'Allow to merge if pipeline is skipped' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of tags for a project' + optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :topics instead' + optional :topics, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of topics for a project' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :avatar, type: File, desc: 'Avatar image for project' # rubocop:disable Scalability/FileUploads optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' @@ -146,6 +148,7 @@ module API :shared_runners_enabled, :snippets_access_level, :tag_list, + :topics, :visibility, :wiki_access_level, :avatar, @@ -154,6 +157,7 @@ module API :compliance_framework_setting, :packages_enabled, :service_desk_enabled, + :keep_latest_artifact, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 6f25cf507bc..9ec9b5e1e35 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -25,6 +25,7 @@ module API return get_runner_ip unless params['info'].present? attributes_for_keys(%w(name version revision platform architecture), params['info']) + .merge(get_runner_config_from_request) .merge(get_runner_ip) end @@ -33,8 +34,15 @@ module API end def current_runner + token = params[:token] + + if token + ::Gitlab::Database::LoadBalancing::RackMiddleware + .stick_or_unstick(env, :runner, token) + end + strong_memoize(:current_runner) do - ::Ci::Runner.find_by_token(params[:token].to_s) + ::Ci::Runner.find_by_token(token.to_s) end end @@ -64,8 +72,15 @@ module API end def current_job + id = params[:id] + + if id + ::Gitlab::Database::LoadBalancing::RackMiddleware + .stick_or_unstick(env, :build, id) + end + strong_memoize(:current_job) do - ::Ci::Build.find_by_id(params[:id]) + ::Ci::Build.find_by_id(id) end end @@ -91,6 +106,12 @@ module API def track_ci_minutes_usage!(_build, _runner) # noop: overridden in EE end + + private + + def get_runner_config_from_request + { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } + end end end end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index d123db8e3df..ca13ea0789a 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -777,41 +777,41 @@ module API ::Integrations::Asana, ::Integrations::Assembla, ::Integrations::Bamboo, + ::Integrations::Bugzilla, + ::Integrations::Buildkite, ::Integrations::Campfire, ::Integrations::Confluence, + ::Integrations::CustomIssueTracker, ::Integrations::Datadog, + ::Integrations::Discord, + ::Integrations::DroneCi, ::Integrations::EmailsOnPush, - ::BugzillaService, - ::BuildkiteService, - ::CustomIssueTrackerService, - ::DiscordService, - ::DroneCiService, - ::EwmService, - ::ExternalWikiService, - ::FlowdockService, - ::HangoutsChatService, - ::IrkerService, - ::JenkinsService, - ::JiraService, - ::MattermostSlashCommandsService, - ::SlackSlashCommandsService, - ::PackagistService, - ::PipelinesEmailService, - ::PivotaltrackerService, - ::PrometheusService, - ::PushoverService, - ::RedmineService, - ::YoutrackService, - ::SlackService, - ::MattermostService, - ::MicrosoftTeamsService, - ::TeamcityService + ::Integrations::Ewm, + ::Integrations::ExternalWiki, + ::Integrations::Flowdock, + ::Integrations::HangoutsChat, + ::Integrations::Irker, + ::Integrations::Jenkins, + ::Integrations::Jira, + ::Integrations::Mattermost, + ::Integrations::MattermostSlashCommands, + ::Integrations::MicrosoftTeams, + ::Integrations::Packagist, + ::Integrations::PipelinesEmail, + ::Integrations::Pivotaltracker, + ::Integrations::Pushover, + ::Integrations::Redmine, + ::Integrations::Slack, + ::Integrations::SlackSlashCommands, + ::Integrations::Teamcity, + ::Integrations::Youtrack, + ::PrometheusService ] end def self.development_service_classes [ - ::MockCiService, + ::Integrations::MockCi, ::MockMonitoringService ] end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index e16149185c9..ee0ddccc8d4 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -10,8 +10,6 @@ module API api_endpoint = env['api.endpoint'] feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s - header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category - Gitlab::ApplicationContext.push( user: -> { actor&.user }, project: -> { project }, @@ -169,18 +167,15 @@ module API end # - # Get a ssh key using the fingerprint + # Check whether an SSH key is known to GitLab # - # rubocop: disable CodeReuse/ActiveRecord get '/authorized_keys', feature_category: :source_code_management do - fingerprint = params.fetch(:fingerprint) do - Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint - end - key = Key.find_by(fingerprint: fingerprint) + fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint + + key = Key.find_by_fingerprint(fingerprint) not_found!('Key') if key.nil? present key, with: Entities::SSHKey end - # rubocop: enable CodeReuse/ActiveRecord # # Discover user by ssh key, user id or username diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 0d562cc18f8..46d8c0c958d 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -23,6 +23,7 @@ module API requires :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma' requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' end post ":id/invitations" do params[:source] = find_source(source_type, params[:id]) diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index cf65bfdfd0e..723a5b0fa3a 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -3,7 +3,6 @@ module API class Jobs < ::API::Base include PaginationParams - before { authenticate! } resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 945cdf3edb2..3580a7b5e24 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -11,7 +11,11 @@ module API optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' end post '/lint' do - unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? + if Feature.enabled?(:security_ci_lint_authorization) + unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? + else + unauthorized! if Gitlab::CurrentSettings.signup_disabled? && current_user.nil? + end result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 22f7b07809b..9e5705abe88 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -24,8 +24,6 @@ module API helpers do def path_exists?(path) - # return true when FF disabled so that processing the request is not stopped - return true unless Feature.enabled?(:check_maven_path_first, default_enabled: :yaml) return false if path.blank? Packages::Maven::Metadatum.with_path(path) @@ -132,7 +130,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: project, namespace: project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end end @@ -172,7 +170,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: package.project, namespace: package.project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -210,7 +208,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -266,7 +264,7 @@ module API when 'md5' '' else - track_package_event('push_package', :maven) if jar_file?(format) + track_package_event('push_package', :maven, user: current_user, project: user_project, namespace: user_project.namespace) if jar_file?(format) file_params = { file: params[:file], diff --git a/lib/api/members.rb b/lib/api/members.rb index a1a733ea7ae..0956806da5b 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -93,6 +93,7 @@ module API requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' end # rubocop: disable CodeReuse/ActiveRecord post ":id/members" do @@ -116,6 +117,7 @@ module API not_allowed! # This currently can only be reached in EE elsif member.valid? && member.persisted? present_members(member) + Gitlab::Tracking.event(::Members::CreateService.name, 'create_member', label: params[:invite_source], property: 'existing_user', user: current_user) else render_validation_error!(member) end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 931d2322c98..a9617482557 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -436,14 +436,11 @@ module API mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params.has_key?(:remove_source_branch) mr_params = convert_parameters_from_legacy_format(mr_params) + mr_params[:use_specialized_service] = true - service = if mr_params.one? && (mr_params.keys & %i[assignee_id assignee_ids]).one? - ::MergeRequests::UpdateAssigneesService - else - ::MergeRequests::UpdateService - end - - merge_request = service.new(project: user_project, current_user: current_user, params: mr_params).execute(merge_request) + merge_request = ::MergeRequests::UpdateService + .new(project: user_project, current_user: current_user, params: mr_params) + .execute(merge_request) handle_merge_request_errors!(merge_request) diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 887084dc9ae..7ff4439ce04 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -32,7 +32,7 @@ module API package_file = ::Packages::PackageFileFinder .new(package, params[:file_name]).execute! - track_package_event('pull_package', package, category: 'API::NpmPackages') + track_package_event('pull_package', package, category: 'API::NpmPackages', project: project, namespace: project.namespace) present_carrierwave_file!(package_file.file) end @@ -48,7 +48,7 @@ module API put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do authorize_create_package!(project) - track_package_event('push_package', :npm, category: 'API::NpmPackages') + track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace) created_package = ::Packages::Npm::CreatePackageService .new(project, current_user, params.merge(build: current_authenticated_job)).execute diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index a80de06d6b0..eb55e4cbf70 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -38,6 +38,10 @@ module API def require_authenticated! unauthorized! unless current_user end + + def snowplow_gitlab_standard_context + { namespace: find_authorized_group! } + end end params do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 73ecc140959..5bae08d4dae 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -36,6 +36,10 @@ module API def project_or_group authorized_user_project end + + def snowplow_gitlab_standard_context + { project: authorized_user_project, namespace: authorized_user_project.namespace } + end end params do @@ -69,7 +73,7 @@ module API package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) .execute - track_package_event('push_package', :nuget, category: 'API::NugetPackages') + track_package_event('push_package', :nuget, category: 'API::NugetPackages', user: current_user, project: package.project, namespace: package.project.namespace) ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker @@ -118,7 +122,7 @@ module API not_found!('Package') unless package_file - track_package_event('pull_package', :nuget, category: 'API::NugetPackages') + track_package_event('pull_package', :nuget, category: 'API::NugetPackages', project: package_file.project, namespace: package_file.project.namespace) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false present_carrierwave_file!(package_file.file, supports_direct_download: false) diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 2580f7adbc9..28cfa9e3ae0 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -31,7 +31,7 @@ module API user: current_user, subject: user_project ).execute - track_package_event('list_repositories', :container) + track_package_event('list_repositories', :container, user: current_user, project: user_project, namespace: user_project.namespace) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end @@ -46,7 +46,7 @@ module API authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker - track_package_event('delete_repository', :container) + track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted end @@ -63,7 +63,7 @@ module API authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) - track_package_event('list_tags', :container) + track_package_event('list_tags', :container, user: current_user, project: user_project, namespace: user_project.namespace) present paginate(tags), with: Entities::ContainerRegistry::Tag end @@ -92,7 +92,7 @@ module API declared_params.except(:repository_id).merge(container_expiration_policy: false)) # rubocop:enable CodeReuse/Worker - track_package_event('delete_tag_bulk', :container) + track_package_event('delete_tag_bulk', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted end @@ -128,7 +128,7 @@ module API .execute(repository) if result[:status] == :success - track_package_event('delete_tag', :container) + track_package_event('delete_tag', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :ok else diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb new file mode 100644 index 00000000000..58edf51f4f7 --- /dev/null +++ b/lib/api/project_debian_distributions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module API + class ProjectDebianDistributions < ::API::Base + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + after_validation do + require_packages_enabled! + + not_found! unless ::Feature.enabled?(:debian_packages, user_project) + + authorize_read_package! + end + + namespace ':id' do + helpers do + def project_or_group + user_project + end + end + + include ::API::Concerns::Packages::DebianDistributionEndpoints + end + end + end +end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 76b3dea723a..4041e130f9e 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -30,7 +30,11 @@ module API check_rate_limit! :project_download_export, [current_user, user_project] if user_project.export_file_exists? - present_carrierwave_file!(user_project.export_file) + if user_project.export_archive_exists? + present_carrierwave_file!(user_project.export_file) + else + render_api_error!('The project export file is not available yet', 404) + end else render_api_error!('404 Not found or has expired', 404) end diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index babc7b9dd58..276cbe50e42 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -41,7 +41,7 @@ module API declared_params.slice(:order_by, :sort, :package_type, :package_name, :include_versionless, :status) ).execute - present paginate(packages), with: ::API::Entities::Package, user: current_user + present paginate(packages), with: ::API::Entities::Package, user: current_user, namespace: user_project.root_ancestor end desc 'Get a single project package' do @@ -55,7 +55,7 @@ module API package = ::Packages::PackageFinder .new(user_project, params[:package_id]).execute - present package, with: ::API::Entities::Package, user: current_user + present package, with: ::API::Entities::Package, user: current_user, namespace: user_project.root_ancestor end desc 'Remove a package' do diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 899984fe0ba..084492fd503 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -75,7 +75,7 @@ module API snippet_params = process_create_params(declared_params(include_missing: false)) - service_response = ::Snippets::CreateService.new(user_project, current_user, snippet_params).execute + service_response = ::Snippets::CreateService.new(project: user_project, current_user: current_user, params: snippet_params).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -116,7 +116,7 @@ module API snippet_params = process_update_params(declared_params(include_missing: false)) - service_response = ::Snippets::UpdateService.new(user_project, current_user, snippet_params).execute(snippet) + service_response = ::Snippets::UpdateService.new(project: user_project, current_user: current_user, params: snippet_params).execute(snippet) snippet = service_response.payload[:snippet] if service_response.success? diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 5d6f67ccbae..acf9bfece65 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -26,7 +26,7 @@ module API use :pagination end get ':id/templates/:type' do - templates = TemplateFinder.all_template_names_array(user_project, params[:type]) + templates = TemplateFinder.all_template_names(user_project, params[:type]).values.flatten present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 4e8786fbe1f..83c335a3248 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -61,7 +61,7 @@ module API # Temporarily introduced for upload API: https://gitlab.com/gitlab-org/gitlab/-/issues/325788 def project_attachment_size(user_project) return PROJECT_ATTACHMENT_SIZE_EXEMPT if exempt_from_global_attachment_size?(user_project) - return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project) + return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project, default_enabled: :yaml) PROJECT_ATTACHMENT_SIZE_EXEMPT end @@ -234,6 +234,7 @@ module API params do optional :name, type: String, desc: 'The name of the project' optional :path, type: String, desc: 'The path of the repository' + optional :default_branch, type: String, desc: 'The default branch of the project' at_least_one_of :name, :path use :optional_create_project_params use :create_params @@ -660,6 +661,18 @@ module API render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) end end + + desc 'Show the storage information' do + success Entities::ProjectRepositoryStorage + end + params do + requires :id, type: String, desc: 'ID of a project' + end + get ':id/storage', feature_category: :projects do + authenticated_as_admin! + + present user_project, with: Entities::ProjectRepositoryStorage, current_user: current_user + end end end end diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 73b2f658825..7c5f8bb4d99 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -28,6 +28,73 @@ module API require_packages_enabled! end + helpers do + params :package_download do + requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true + requires :sha256, type: String, desc: 'The PyPi package sha256 check sum' + end + + params :package_name do + requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' + end + end + + params do + requires :id, type: Integer, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + after_validation do + unauthorized_user_group! + end + + namespace ':id/-/packages/pypi' do + params do + use :package_download + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'files/:sha256/*file_identifier' do + group = unauthorized_user_group! + + filename = "#{params[:file_identifier]}.#{params[:format]}" + package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute + package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute + + track_package_event('pull_package', :pypi) + + present_carrierwave_file!(package_file.file, supports_direct_download: true) + end + + desc 'The PyPi Simple Endpoint' do + detail 'This feature was introduced in GitLab 12.10' + end + + params do + use :package_name + end + + # An Api entry point but returns an HTML file instead of JSON. + # PyPi simple API returns the package descriptor as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple/*package_name', format: :txt do + group = find_authorized_group! + authorize_read_package!(group) + + track_package_event('list_package', :pypi) + + packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute! + presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) + + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary + + body presenter.body + end + end + end + params do requires :id, type: Integer, desc: 'The ID of a project' end @@ -43,8 +110,7 @@ module API end params do - requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true - requires :sha256, type: String, desc: 'The PyPi package sha256 check sum' + use :package_download end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth @@ -55,7 +121,7 @@ module API package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute - track_package_event('pull_package', :pypi) + track_package_event('pull_package', :pypi, project: project, namespace: project.namespace) present_carrierwave_file!(package_file.file, supports_direct_download: true) end @@ -65,7 +131,7 @@ module API end params do - requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' + use :package_name end # An Api entry point but returns an HTML file instead of JSON. @@ -74,7 +140,7 @@ module API get 'simple/*package_name', format: :txt do authorize_read_package!(authorized_user_project) - track_package_event('list_package', :pypi) + track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace) packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute! presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) @@ -105,7 +171,7 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) - track_package_event('push_package', :pypi) + track_package_event('push_package', :pypi, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) ::Packages::Pypi::CreatePackageService .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 1d17148e0df..d7f9c584c67 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -70,7 +70,7 @@ module API user_project, params[:file_name] ).last! - track_package_event('pull_package', :rubygems) + track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace) present_carrierwave_file!(package_file.file) end @@ -97,7 +97,7 @@ module API authorize_upload!(user_project) bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size) - track_package_event('push_package', :rubygems) + track_package_event('push_package', :rubygems, user: current_user, project: user_project, namespace: user_project.namespace) package_file = nil diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 372bc7b3d8f..b4f8320cb74 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -46,7 +46,7 @@ module API optional :asset_proxy_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" - optional :default_ci_config_path, type: String, desc: 'The instance default CI configuration path for new projects' + optional :default_ci_config_path, type: String, desc: 'The instance default CI/CD configuration file and path for new projects' optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 52b597fb788..b506192fe1c 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -84,7 +84,7 @@ module API attrs = process_create_params(declared_params(include_missing: false)) - service_response = ::Snippets::CreateService.new(nil, current_user, attrs).execute + service_response = ::Snippets::CreateService.new(project: nil, current_user: current_user, params: attrs).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -126,7 +126,7 @@ module API attrs = process_update_params(declared_params(include_missing: false)) - service_response = ::Snippets::UpdateService.new(nil, current_user, attrs).execute(snippet) + service_response = ::Snippets::UpdateService.new(project: nil, current_user: current_user, params: attrs).execute(snippet) snippet = service_response.payload[:snippet] diff --git a/lib/api/tags.rb b/lib/api/tags.rb index e77d7e34de3..6c8e2c69a6d 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -51,35 +51,22 @@ module API end desc 'Create a new repository tag' do - detail 'This optional release_description parameter was deprecated in GitLab 11.7.' success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' requires :ref, type: String, desc: 'The commit sha or branch name' optional :message, type: String, desc: 'Specifying a message creates an annotated tag' - optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end post ':id/repository/tags', :release_orchestration do + deprecate_release_notes unless params[:release_description].blank? + authorize_admin_tag result = ::Tags::CreateService.new(user_project, current_user) .execute(params[:tag_name], params[:ref], params[:message]) if result[:status] == :success - # Release creation with Tags API was deprecated in GitLab 11.7 - if params[:release_description].present? - release_create_params = { - tag: params[:tag_name], - name: params[:tag_name], # Name can be specified in new API - description: params[:release_description] - } - - ::Releases::CreateService - .new(user_project, current_user, release_create_params) - .execute - end - present result[:tag], with: Entities::Tag, project: user_project @@ -109,74 +96,6 @@ module API end end end - - desc 'Add a release note to a tag' do - detail 'This feature was deprecated in GitLab 11.7.' - success Entities::TagRelease - end - params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag - requires :description, type: String, desc: 'Release notes with markdown support' - end - post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do - authorize_create_release! - - ## - # Legacy API does not support tag auto creation. - not_found!('Tag') unless user_project.repository.find_tag(params[:tag]) - - release_create_params = { - tag: params[:tag], - name: params[:tag], # Name can be specified in new API - description: params[:description] - } - - result = ::Releases::CreateService - .new(user_project, current_user, release_create_params) - .execute - - if result[:status] == :success - present result[:release], with: Entities::TagRelease - else - render_api_error!(result[:message], result[:http_status]) - end - end - - desc "Update a tag's release note" do - detail 'This feature was deprecated in GitLab 11.7.' - success Entities::TagRelease - end - params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag - requires :description, type: String, desc: 'Release notes with markdown support' - end - put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do - authorize_update_release! - - result = ::Releases::UpdateService - .new(user_project, current_user, declared_params(include_missing: false)) - .execute - - if result[:status] == :success - present result[:release], with: Entities::TagRelease - else - render_api_error!(result[:message], result[:http_status]) - end - end - end - - helpers do - def authorize_create_release! - authorize! :create_release, user_project - end - - def authorize_update_release! - authorize! :update_release, release - end - - def release - @release ||= user_project.releases.find_by_tag(params[:tag]) - end end end end diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 34e77e09800..aa59b6a4fee 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -124,7 +124,7 @@ module API end get do - track_package_event('pull_package', :terraform_module) + track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace, user: current_user) present_carrierwave_file!(package_file.file) end @@ -183,7 +183,7 @@ module API render_api_error!(result[:message], result[:http_status]) if result[:status] == :error - track_package_event('push_package', :terraform_module) + track_package_event('push_package', :terraform_module, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) created! rescue ObjectStorage::RemoteStoreError => e diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 3148c56339a..37fe540cde1 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -69,10 +69,7 @@ module API def feature_flags return [] unless unleash_app_name.present? - legacy_flags = Operations::FeatureFlagScope.for_unleash_client(project, unleash_app_name) - new_version_flags = Operations::FeatureFlag.for_unleash_client(project, unleash_app_name) - - legacy_flags + new_version_flags + Operations::FeatureFlag.for_unleash_client(project, unleash_app_name) end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 565a3544da2..2608fb87e22 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -241,7 +241,7 @@ module API authenticated_as_admin! params = declared_params(include_missing: false) - user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true) + user = ::Users::AuthorizedCreateService.new(current_user, params).execute if user.persisted? present user, with: Entities::UserWithAdmin, current_user: current_user @@ -1025,7 +1025,9 @@ module API detail 'This feature was introduced in GitLab 13.10.' end params do - requires :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' + optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' + optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' + at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs end put "preferences", feature_category: :users do authenticate! @@ -1043,6 +1045,14 @@ module API end end + desc "Get the current user's preferences" do + success Entities::UserPreferences + detail 'This feature was introduced in GitLab 14.0.' + end + get "preferences", feature_category: :users do + present current_user.user_preference, with: Entities::UserPreferences + end + desc 'Get a single email address owned by the currently authenticated user' do success Entities::Email end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb new file mode 100644 index 00000000000..cfd3d463f9e --- /dev/null +++ b/lib/backup/gitaly_backup.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Backup + # Backup and restores repositories using gitaly-backup + class GitalyBackup + def initialize(progress) + @progress = progress + end + + def start(type) + raise Error, 'already started' if started? + + command = case type + when :create + 'create' + when :restore + 'restore' + else + raise Error, "unknown backup type: #{type}" + end + + @read_io, @write_io = IO.pipe + @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress) + end + + def wait + return unless started? + + @write_io.close + Process.wait(@pid) + status = $? + + @pid = nil + + raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0 + end + + def enqueue(container, repo_type) + raise Error, 'not started' unless started? + + repository = repo_type.repository_for(container) + + @write_io.puts({ + storage_name: repository.storage, + relative_path: repository.relative_path, + gl_project_path: repository.gl_project_path, + always_create: repo_type.project? + }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) + end + + private + + attr_reader :progress + + def started? + @pid.present? + end + + def backup_repos_path + File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories')) + end + + def bin_path + File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup')) + end + end +end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb new file mode 100644 index 00000000000..53f1de40509 --- /dev/null +++ b/lib/backup/gitaly_rpc_backup.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Backup + # Backup and restores repositories using the gitaly RPC + class GitalyRpcBackup + def initialize(progress) + @progress = progress + end + + def start(type) + raise Error, 'already started' if @type + + @type = type + case type + when :create + FileUtils.rm_rf(backup_repos_path) + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.mkdir(backup_repos_path, mode: 0700) + when :restore + # no op + else + raise Error, "unknown backup type: #{type}" + end + end + + def wait + @type = nil + end + + def enqueue(container, repository_type) + backup_restore = BackupRestore.new( + progress, + repository_type.repository_for(container), + backup_repos_path + ) + + case @type + when :create + backup_restore.backup + when :restore + backup_restore.restore(always_create: repository_type.project?) + else + raise Error, 'not started' + end + end + + private + + attr_reader :progress + + def backup_repos_path + @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') + end + + class BackupRestore + attr_accessor :progress, :repository, :backup_repos_path + + def initialize(progress, repository, backup_repos_path) + @progress = progress + @repository = repository + @backup_repos_path = backup_repos_path + end + + def backup + progress.puts " * #{display_repo_path} ... " + + if repository.empty? + progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) + return + end + + FileUtils.mkdir_p(repository_backup_path) + + repository.bundle_to_disk(path_to_bundle) + repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue StandardError => e + progress.puts "[Failed] backing up #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + def restore(always_create: false) + progress.puts " * #{display_repo_path} ... " + + repository.remove rescue nil + + if File.exist?(path_to_bundle) + repository.create_from_bundle(path_to_bundle) + restore_custom_hooks + elsif always_create + repository.create_repository + end + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue StandardError => e + progress.puts "[Failed] restoring #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + private + + def display_repo_path + "#{repository.full_path} (#{repository.disk_path})" + end + + def repository_backup_path + @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) + end + + def path_to_bundle + @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') + end + + def restore_custom_hooks + return unless File.exist?(custom_hooks_tar) + + repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + end + + def custom_hooks_tar + File.join(repository_backup_path, "custom_hooks.tar") + end + end + end +end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index b1231eebfcc..80d23c1eb7f 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -4,17 +4,16 @@ require 'yaml' module Backup class Repositories - attr_reader :progress - - def initialize(progress) + def initialize(progress, strategy:) @progress = progress + @strategy = strategy end def dump(max_concurrency:, max_storage_concurrency:) - prepare + strategy.start(:create) if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive + return enqueue_consecutive end check_valid_storages! @@ -25,7 +24,7 @@ module Backup threads = Gitlab.config.repositories.storages.keys.map do |storage| Thread.new do Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) rescue StandardError => e errors << e end @@ -37,32 +36,24 @@ module Backup end raise errors.pop unless errors.empty? + ensure + strategy.wait end def restore - restore_project_repositories - restore_snippets + strategy.start(:restore) + enqueue_consecutive + + ensure + strategy.wait + cleanup_snippets_without_repositories restore_object_pools end private - def restore_project_repositories - Project.find_each(batch_size: 1000) do |project| - restore_repository(project, Gitlab::GlRepository::PROJECT) - restore_repository(project, Gitlab::GlRepository::WIKI) - restore_repository(project, Gitlab::GlRepository::DESIGN) - end - end - - def restore_snippets - invalid_ids = Snippet.find_each(batch_size: 1000) - .map { |snippet| restore_snippet_repository(snippet) } - .compact - - cleanup_snippets_without_repositories(invalid_ids) - end + attr_reader :progress, :strategy def check_valid_storages! repository_storage_klasses.each do |klass| @@ -76,32 +67,22 @@ module Backup [ProjectRepository, SnippetRepository] end - def backup_repos_path - @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) + def enqueue_consecutive + enqueue_consecutive_projects + enqueue_consecutive_snippets end - def dump_consecutive - dump_consecutive_projects - dump_consecutive_snippets - end - - def dump_consecutive_projects + def enqueue_consecutive_projects project_relation.find_each(batch_size: 1000) do |project| - dump_project(project) + enqueue_project(project) end end - def dump_consecutive_snippets - Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) } + def enqueue_consecutive_snippets + Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end - def dump_storage(storage, semaphore, max_storage_concurrency:) + def enqueue_storage(storage, semaphore, max_storage_concurrency:) errors = Queue.new queue = InterlockSizedQueue.new(1) @@ -114,7 +95,7 @@ module Backup end begin - dump_container(container) + enqueue_container(container) rescue StandardError => e errors << e break @@ -136,23 +117,23 @@ module Backup end end - def dump_container(container) + def enqueue_container(container) case container when Project - dump_project(container) + enqueue_project(container) when Snippet - dump_snippet(container) + enqueue_snippet(container) end end - def dump_project(project) - backup_repository(project, Gitlab::GlRepository::PROJECT) - backup_repository(project, Gitlab::GlRepository::WIKI) - backup_repository(project, Gitlab::GlRepository::DESIGN) + def enqueue_project(project) + strategy.enqueue(project, Gitlab::GlRepository::PROJECT) + strategy.enqueue(project, Gitlab::GlRepository::WIKI) + strategy.enqueue(project, Gitlab::GlRepository::DESIGN) end - def dump_snippet(snippet) - backup_repository(snippet, Gitlab::GlRepository::SNIPPET) + def enqueue_snippet(snippet) + strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET) end def enqueue_records_for_storage(storage, queue, errors) @@ -181,22 +162,6 @@ module Backup Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) end - def backup_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).backup - end - - def restore_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).restore(always_create: type.project?) - end - def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." @@ -214,99 +179,22 @@ module Backup end end - def restore_snippet_repository(snippet) - restore_repository(snippet, Gitlab::GlRepository::SNIPPET) - - response = Snippets::RepositoryValidationService.new(nil, snippet).execute - - if response.error? - snippet.repository.remove - - progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - - snippet.id - else - nil - end - end - # Snippets without a repository should be removed because they failed to import # due to having invalid repositories - def cleanup_snippets_without_repositories(ids) - Snippet.id_in(ids).delete_all - end + def cleanup_snippets_without_repositories + invalid_snippets = [] - class BackupRestore - attr_accessor :progress, :repository, :backup_repos_path + Snippet.find_each(batch_size: 1000).each do |snippet| + response = Snippets::RepositoryValidationService.new(nil, snippet).execute + next if response.success? - def initialize(progress, repository, backup_repos_path) - @progress = progress - @repository = repository - @backup_repos_path = backup_repos_path - end - - def backup - progress.puts " * #{display_repo_path} ... " - - if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) - return - end - - FileUtils.mkdir_p(repository_backup_path) - - repository.bundle_to_disk(path_to_bundle) - repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] backing up #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - def restore(always_create: false) - progress.puts " * #{display_repo_path} ... " - - repository.remove rescue nil - - if File.exist?(path_to_bundle) - repository.create_from_bundle(path_to_bundle) - restore_custom_hooks - elsif always_create - repository.create_repository - end - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] restoring #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - private - - def display_repo_path - "#{repository.full_path} (#{repository.disk_path})" - end - - def repository_backup_path - @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) - end - - def path_to_bundle - @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') - end - - def restore_custom_hooks - return unless File.exist?(custom_hooks_tar) + snippet.repository.remove + progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + invalid_snippets << snippet.id end - def custom_hooks_tar - File.join(repository_backup_path, "custom_hooks.tar") - end + Snippet.id_in(invalid_snippets).delete_all end class InterlockSizedQueue < SizedQueue diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb index 3f775abb185..b2eaeb69f61 100644 --- a/lib/banzai/filter/base_relative_link_filter.rb +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -13,18 +13,12 @@ module Banzai protected def linkable_attributes - if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) - # Nokorigi Nodeset#search performs badly for documents with many nodes - # - # Here we store fetched attributes in the shared variable "result" - # This variable is passed through the chain of filters and can be - # accessed by them - result[:linkable_attributes] ||= fetch_linkable_attributes - else - strong_memoize(:linkable_attributes) do - fetch_linkable_attributes - end - end + # Nokorigi Nodeset#search performs badly for documents with many nodes + # + # Here we store fetched attributes in the shared variable "result" + # This variable is passed through the chain of filters and can be + # accessed by them + result[:linkable_attributes] ||= fetch_linkable_attributes end def relative_url_root diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index 0c53444681d..8d54d140877 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -30,8 +30,6 @@ module Banzai LITERAL_KEYWORD = 'cmliteral' def call - return @text unless Feature.enabled?(:honor_escaped_markdown, context[:group] || context[:project]&.group) - @text.gsub(ASCII_PUNCTUATION) do |match| # The majority of markdown does not have literals. If none # are found, we can bypass the post filter diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb index bf6b3e47d3b..12afece6e53 100644 --- a/lib/banzai/filter/references/label_reference_filter.rb +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -8,21 +8,57 @@ module Banzai self.reference_type = :label self.object_class = Label + def parent_records(parent, ids) + return Label.none unless parent.is_a?(Project) || parent.is_a?(Group) + + labels = find_labels(parent) + label_ids = ids.map {|y| y[:label_id]}.compact + label_names = ids.map {|y| y[:label_name]}.compact + id_relation = labels.where(id: label_ids) + label_relation = labels.where(title: label_names) + + Label.from_union([id_relation, label_relation]) + end + def find_object(parent_object, id) - find_labels(parent_object).find(id) + key = reference_cache.records_per_parent[parent_object].keys.find do |k| + k[:label_id] == id[:label_id] || k[:label_name] == id[:label_name] + end + + reference_cache.records_per_parent[parent_object][key] if key + end + + # Transform a symbol extracted from the text to a meaningful value + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `parse_symbol(ref) == record_identifier(record)`. + # + # This contract is slightly broken here, as we only have either the label_id + # or the label_name, but not both. But below, we have both pieces of information. + # But it's accounted for in `find_object` + def parse_symbol(symbol, match_data) + { label_id: match_data[:label_id]&.to_i, label_name: match_data[:label_name]&.tr('"', '') } + end + + # We assume that most classes are identifying records by ID. + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`. + # See note in `parse_symbol` above + def record_identifier(record) + { label_id: record.id, label_name: record.title } end def references_in(text, pattern = Label.reference_pattern) labels = {} - unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| - namespace = $~[:namespace] - project = $~[:project] - project_path = reference_cache.full_project_path(namespace, project) - label = find_label_cached(project_path, $~[:label_id], $~[:label_name]) - - if label - labels[label.id] = yield match, label.id, project, namespace, $~ - "#{REFERENCE_PLACEHOLDER}#{label.id}" + + unescaped_html = unescape_html_entities(text).gsub(pattern).with_index do |match, index| + ident = identifier($~) + label = yield match, ident, $~[:project], $~[:namespace], $~ + + if label != match + labels[index] = label + "#{REFERENCE_PLACEHOLDER}#{index}" else match end @@ -33,20 +69,6 @@ module Banzai escape_with_placeholders(unescaped_html, labels) end - def find_label_cached(parent_ref, label_id, label_name) - cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do - find_label(parent_ref, label_id, label_name) - end - end - - def find_label(parent_ref, label_id, label_name) - parent = parent_from_ref(parent_ref) - return unless parent - - label_params = label_params(label_id, label_name) - find_labels(parent).find_by(label_params) - end - def find_labels(parent) params = if parent.is_a?(Group) { group_id: parent.id, @@ -60,21 +82,6 @@ module Banzai LabelsFinder.new(nil, params).execute(skip_authorization: true) end - # Parameters to pass to `Label.find_by` based on the given arguments - # - # id - Integer ID to pass. If present, returns {id: id} - # name - String name to pass. If `id` is absent, finds by name without - # surrounding quotes. - # - # Returns a Hash. - def label_params(id, name) - if name - { name: name.tr('"', '') } - else - { id: id.to_i } - end - end - def url_for_object(label, parent) label_url_method = if context[:label_url_method] @@ -121,6 +128,14 @@ module Banzai presenter = object.present(issuable_subject: project || group) LabelsHelper.label_tooltip_title(presenter) end + + def parent + project || group + end + + def requires_unescaping? + true + end end end end diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb index ab0c74e00d9..24b8b4984cd 100644 --- a/lib/banzai/filter/references/reference_cache.rb +++ b/lib/banzai/filter/references/reference_cache.rb @@ -29,15 +29,15 @@ module Banzai refs = Hash.new { |hash, key| hash[key] = Set.new } nodes.each do |node| - node.to_html.scan(regex) do - path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end + prepare_node_for_scan(node).scan(regex) do + parent_path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end ident = filter.identifier($~) - refs[path] << ident if ident + refs[parent_path] << ident if ident end end @@ -55,9 +55,23 @@ module Banzai @per_reference ||= {} @per_reference[parent_type] ||= begin - refs = references_per_parent.keys.to_set + refs = references_per_parent.keys + parent_ref = {} - find_for_paths(refs.to_a).index_by(&:full_path) + # if we already have a parent, no need to query it again + refs.each do |ref| + next unless ref + + if context[:project]&.full_path == ref + parent_ref[ref] = context[:project] + elsif context[:group]&.full_path == ref + parent_ref[ref] = context[:group] + end + + refs -= [ref] if parent_ref[ref] + end + + find_for_paths(refs).index_by(&:full_path).merge(parent_ref) end end @@ -87,7 +101,7 @@ module Banzai @_records_per_project[filter.object_class.to_s.underscore] end - def relation_for_paths(paths) + def objects_for_paths(paths) klass = parent_type.to_s.camelize.constantize result = klass.where_full_path_in(paths) return result if parent_type == :group @@ -102,7 +116,7 @@ module Banzai to_query = paths - cache.keys unless to_query.empty? - records = relation_for_paths(to_query) + records = objects_for_paths(to_query) found = [] records.each do |record| @@ -119,7 +133,7 @@ module Banzai cache.slice(*paths).values.compact else - relation_for_paths(paths) + objects_for_paths(paths) end end @@ -170,6 +184,16 @@ module Banzai def refs_cache Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end + + def prepare_node_for_scan(node) + html = node.to_html + + filter.requires_unescaping? ? unescape_html_entities(html) : html + end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end end end end diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index 58436f4505e..6c2c993cc01 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -109,6 +109,10 @@ module Banzai context[:group] end + def requires_unescaping? + false + end + private # Returns a data attribute String to attach to a reference link diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index ceb7547a85d..2572481c8fc 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -15,16 +15,10 @@ module Banzai def call return doc if context[:system_note] - if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) - # We exclude processed upload links from the linkable attributes to - # prevent further modifications by RepositoryLinkFilter - linkable_attributes.reject! do |attr| - process_link_to_upload_attr(attr) - end - else - linkable_attributes.each do |attr| - process_link_to_upload_attr(attr) - end + # We exclude processed upload links from the linkable attributes to + # prevent further modifications by RepositoryLinkFilter + linkable_attributes.reject! do |attr| + process_link_to_upload_attr(attr) end doc diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index c86d5f08ded..17a73f29afb 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -9,7 +9,8 @@ module Banzai Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, - Filter::SyntaxHighlightFilter + Filter::SyntaxHighlightFilter, + Filter::KrokiFilter ] end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 97c7173ac0f..6b1491cc56b 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -66,14 +66,7 @@ module Banzai # These associations are primarily used for checking permissions. # Eager loading these ensures we don't end up running dozens of # queries in this process. - project: [ - { namespace: :owner }, - { group: [:owners, :group_members] }, - :invited_groups, - :project_members, - :project_feature, - :route - ] + project: [:namespace, :project_feature, :route] } ), self.class.data_attribute diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 24bc1a24e09..1664fa1f9ff 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -7,6 +7,19 @@ module Banzai self.reference_type = :merge_request + def nodes_visible_to_user(user, nodes) + return super if Feature.disabled?(:optimize_merge_request_parser, user, default_enabled: :yaml) + + merge_request_nodes = nodes.select { |node| node.has_attribute?(self.class.data_attribute) } + records = projects_for_nodes(merge_request_nodes) + + merge_request_nodes.select do |node| + project = records[node] + + project && can_read_reference?(user, project) + end + end + def records_for_nodes(nodes) @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, @@ -17,27 +30,25 @@ module Banzai # These associations are primarily used for checking permissions. # Eager loading these ensures we don't end up running dozens of # queries in this process. - target_project: [ - { namespace: [:owner, :route] }, - { group: [:owners, :group_members] }, - :invited_groups, - :project_members, - :project_feature, - :route - ] + target_project: [{ namespace: :route }, :project_feature, :route] }), self.class.data_attribute ) end - def can_read_reference?(user, merge_request) + def can_read_reference?(user, object) memo = strong_memoize(:can_read_reference) { {} } - project_id = merge_request.project_id + project_id = object.project_id return memo[project_id] if memo.key?(project_id) - memo[project_id] = can?(user, :read_merge_request_iid, merge_request.project) + memo[project_id] = can?(user, :read_merge_request_iid, object) + end + + def projects_for_nodes(nodes) + @projects_for_nodes ||= + grouped_objects_for_nodes(nodes, Project.includes(:project_feature, :group, :namespace), 'data-project') end end end diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb index b067431aeae..ca549c4be14 100644 --- a/lib/bulk_imports/clients/graphql.rb +++ b/lib/bulk_imports/clients/graphql.rb @@ -25,7 +25,7 @@ module BulkImports delegate :query, :parse, :execute, to: :client - def initialize(url: Gitlab::COM_URL, token: nil) + def initialize(url: Gitlab::Saas.com_url, token: nil) @url = Gitlab::Utils.append_path(url, '/api/graphql') @token = token @client = Graphlient::Client.new( diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index c89679f63b5..c5f12d8c2ba 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -2,7 +2,7 @@ module BulkImports module Clients - class Http + class HTTP API_VERSION = 'v4' DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 30 @@ -18,25 +18,19 @@ module BulkImports end def get(resource, query = {}) - with_error_handling do - Gitlab::HTTP.get( - resource_url(resource), - headers: request_headers, - follow_redirects: false, - query: query.reverse_merge(request_query) - ) - end + request(:get, resource, query: query.reverse_merge(request_query)) end def post(resource, body = {}) - with_error_handling do - Gitlab::HTTP.post( - resource_url(resource), - headers: request_headers, - follow_redirects: false, - body: body - ) - end + request(:post, resource, body: body) + end + + def head(resource) + request(:head, resource) + end + + def stream(resource, &block) + request(:get, resource, stream_body: true, &block) end def each_page(method, resource, query = {}, &block) @@ -55,8 +49,36 @@ module BulkImports end end + def resource_url(resource) + Gitlab::Utils.append_path(api_url, resource) + end + private + # rubocop:disable GitlabSecurity/PublicSend + def request(method, resource, options = {}, &block) + with_error_handling do + Gitlab::HTTP.public_send( + method, + resource_url(resource), + request_options(options), + &block + ) + end + end + # rubocop:enable GitlabSecurity/PublicSend + + def request_options(options) + default_options.merge(options) + end + + def default_options + { + headers: request_headers, + follow_redirects: false + } + end + def request_query { page: @page, @@ -88,10 +110,6 @@ module BulkImports def api_url Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}") end - - def resource_url(resource) - Gitlab::Utils.append_path(api_url, resource) - end end end end diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb new file mode 100644 index 00000000000..79d626001a0 --- /dev/null +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Extractors + class NdjsonExtractor + include Gitlab::ImportExport::CommandLineUtil + include Gitlab::Utils::StrongMemoize + + EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}" + + def initialize(relation:) + @relation = relation + @tmp_dir = Dir.mktmpdir + end + + def extract(context) + download_service(tmp_dir, context).execute + decompression_service(tmp_dir).execute + relations = ndjson_reader(tmp_dir).consume_relation('', relation) + + BulkImports::Pipeline::ExtractedData.new(data: relations) + end + + def remove_tmp_dir + FileUtils.remove_entry(tmp_dir) + end + + private + + attr_reader :relation, :tmp_dir + + def filename + @filename ||= "#{relation}.ndjson.gz" + end + + def download_service(tmp_dir, context) + @download_service ||= BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: relative_resource_url(context), + dir: tmp_dir, + filename: filename + ) + end + + def decompression_service(tmp_dir) + @decompression_service ||= BulkImports::FileDecompressionService.new( + dir: tmp_dir, + filename: filename + ) + end + + def ndjson_reader(tmp_dir) + @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmp_dir) + end + + def relative_resource_url(context) + strong_memoize(:relative_resource_url) do + resource = context.portable.class.name.downcase.pluralize + encoded_full_path = context.entity.encoded_source_full_path + + EXPORT_DOWNLOAD_URL_PATH % { resource: resource, full_path: encoded_full_path, relation: relation } + end + end + end + end + end +end diff --git a/lib/bulk_imports/common/extractors/rest_extractor.rb b/lib/bulk_imports/common/extractors/rest_extractor.rb index b18e27fd475..2179e0575c5 100644 --- a/lib/bulk_imports/common/extractors/rest_extractor.rb +++ b/lib/bulk_imports/common/extractors/rest_extractor.rb @@ -24,7 +24,7 @@ module BulkImports attr_reader :query def http_client(configuration) - @http_client ||= BulkImports::Clients::Http.new( + @http_client ||= BulkImports::Clients::HTTP.new( uri: configuration.url, token: configuration.access_token, per_page: 100 diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index e5e2b9fdbd4..db5882d49a9 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -17,7 +17,7 @@ module BulkImports private def http_client(configuration) - @http_client ||= BulkImports::Clients::Http.new( + @http_client ||= BulkImports::Clients::HTTP.new( uri: configuration.url, token: configuration.access_token, per_page: 100 diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb deleted file mode 100644 index f957cf0be52..00000000000 --- a/lib/bulk_imports/groups/graphql/get_labels_query.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Graphql - module GetLabelsQuery - extend self - - def to_s - <<-'GRAPHQL' - query ($full_path: ID!, $cursor: String, $per_page: Int) { - group(fullPath: $full_path) { - labels(first: $per_page, after: $cursor, onlyGroupLabels: true) { - page_info: pageInfo { - next_page: endCursor - has_next_page: hasNextPage - } - nodes { - title - description - color - created_at: createdAt - updated_at: updatedAt - } - } - } - } - GRAPHQL - end - - def variables(context) - { - full_path: context.entity.source_full_path, - cursor: context.tracker.next_page, - per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE - } - end - - def base_path - %w[data group labels] - end - - def data_path - base_path << 'nodes' - end - - def page_info_path - base_path << 'page_info' - end - end - end - end -end diff --git a/lib/bulk_imports/groups/pipelines/boards_pipeline.rb b/lib/bulk_imports/groups/pipelines/boards_pipeline.rb new file mode 100644 index 00000000000..08a0a4abc9f --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/boards_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class BoardsPipeline + include NdjsonPipeline + + relation_name 'boards' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/groups/pipelines/entity_finisher.rb index 1d237bc0f7f..1a709179bf9 100644 --- a/lib/bulk_imports/groups/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/groups/pipelines/entity_finisher.rb @@ -4,31 +4,45 @@ module BulkImports module Groups module Pipelines class EntityFinisher + def self.ndjson_pipeline? + false + end + def initialize(context) @context = context + @entity = @context.entity + @trackers = @entity.trackers end def run - return if context.entity.finished? + return if entity.finished? || entity.failed? - context.entity.finish! + if all_other_trackers_failed? + entity.fail_op! + else + entity.finish! + end logger.info( bulk_import_id: context.bulk_import.id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, pipeline_class: self.class.name, - message: 'Entity finished' + message: "Entity #{entity.status_name}" ) end private - attr_reader :context + attr_reader :context, :entity, :trackers def logger @logger ||= Gitlab::Import::Logger.build end + + def all_other_trackers_failed? + trackers.where.not(relation: self.class.name).all? { |tracker| tracker.failed? } # rubocop: disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb index 0dc4a968b84..1dd74c10b65 100644 --- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb @@ -4,16 +4,11 @@ module BulkImports module Groups module Pipelines class LabelsPipeline - include Pipeline + include NdjsonPipeline - extractor BulkImports::Common::Extractors::GraphqlExtractor, - query: BulkImports::Groups::Graphql::GetLabelsQuery + relation_name 'labels' - transformer Common::Transformers::ProhibitedAttributesTransformer - - def load(context, data) - Labels::CreateService.new(data).execute(group: context.group) - end + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation end end end diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb index 9b2be30735c..b2bd14952e7 100644 --- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb @@ -4,26 +4,11 @@ module BulkImports module Groups module Pipelines class MilestonesPipeline - include Pipeline + include NdjsonPipeline - extractor BulkImports::Common::Extractors::GraphqlExtractor, - query: BulkImports::Groups::Graphql::GetMilestonesQuery + relation_name 'milestones' - transformer Common::Transformers::ProhibitedAttributesTransformer - - def load(context, data) - return unless data - - raise ::BulkImports::Pipeline::NotAllowedError unless authorized? - - context.group.milestones.create!(data) - end - - private - - def authorized? - context.current_user.can?(:admin_milestone, context.group) - end + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation end end end diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb new file mode 100644 index 00000000000..2de06bbcb88 --- /dev/null +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module BulkImports + module NdjsonPipeline + extend ActiveSupport::Concern + + include Pipeline + + included do + ndjson_pipeline! + + def transform(context, data) + relation_hash, relation_index = data + relation_definition = import_export_config.top_relation_tree(relation) + + deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash| + Gitlab::ImportExport::Group::RelationFactory.create( + relation_index: relation_index, + relation_sym: key.to_sym, + relation_hash: hash, + importable: context.portable, + members_mapper: members_mapper, + object_builder: object_builder, + user: context.current_user, + excluded_keys: import_export_config.relation_excluded_keys(key) + ) + end + end + + def load(_, object) + return unless object + + object.save! unless object.persisted? + end + + def deep_transform_relation!(relation_hash, relation_key, relation_definition, &block) + relation_key = relation_key_override(relation_key) + + relation_definition.each do |sub_relation_key, sub_relation_definition| + sub_relation = relation_hash[sub_relation_key] + + next unless sub_relation + + current_item = + if sub_relation.is_a?(Array) + sub_relation + .map { |entry| deep_transform_relation!(entry, sub_relation_key, sub_relation_definition, &block) } + .tap { |entry| entry.compact! } + .presence + else + deep_transform_relation!(sub_relation, sub_relation_key, sub_relation_definition, &block) + end + + if current_item + relation_hash[sub_relation_key] = current_item + else + relation_hash.delete(sub_relation_key) + end + end + + yield(relation_key, relation_hash) + end + + def after_run(_) + extractor.remove_tmp_dir if extractor.respond_to?(:remove_tmp_dir) + end + + def relation_class(relation_key) + relation_key.to_s.classify.constantize + rescue NameError + relation_key.to_s.constantize + end + + def relation_key_override(relation_key) + relation_key_overrides[relation_key.to_sym]&.to_s || relation_key + end + + def relation_key_overrides + "Gitlab::ImportExport::#{portable.class}::RelationFactory::OVERRIDES".constantize + end + + def object_builder + "Gitlab::ImportExport::#{portable.class}::ObjectBuilder".constantize + end + + def relation + self.class.relation + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( + exported_members: [], + user: current_user, + importable: portable + ) + end + end + end +end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index df4f020d6b2..f27818dae18 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -8,8 +8,11 @@ module BulkImports include Runner NotAllowedError = Class.new(StandardError) + ExpiredError = Class.new(StandardError) + FailedError = Class.new(StandardError) CACHE_KEY_EXPIRATION = 2.hours + NDJSON_EXPORT_TIMEOUT = 30.minutes def initialize(context) @context = context @@ -19,6 +22,18 @@ module BulkImports @tracker ||= context.tracker end + def portable + @portable ||= context.portable + end + + def import_export_config + @import_export_config ||= context.import_export_config + end + + def current_user + @current_user ||= context.current_user + end + included do private @@ -111,7 +126,7 @@ module BulkImports options = class_config[:options] if options - class_config[:klass].new(class_config[:options]) + class_config[:klass].new(**class_config[:options]) else class_config[:klass].new end @@ -155,6 +170,22 @@ module BulkImports class_attributes[:abort_on_failure] end + def ndjson_pipeline! + class_attributes[:ndjson_pipeline] = true + end + + def ndjson_pipeline? + class_attributes[:ndjson_pipeline] + end + + def relation_name(name) + class_attributes[:relation_name] = name + end + + def relation + class_attributes[:relation_name] + end + private def add_attribute(sym, klass, options) diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index 3c69c729f36..d753f888671 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -16,6 +16,14 @@ module BulkImports @entity ||= tracker.entity end + def portable + @portable ||= entity.group || entity.project + end + + def import_export_config + @import_export_config ||= ::BulkImports::FileTransfer.config_for(portable) + end + def group @group ||= entity.group end diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb index c9e54b61dd3..0b36c068298 100644 --- a/lib/bulk_imports/pipeline/extracted_data.rb +++ b/lib/bulk_imports/pipeline/extracted_data.rb @@ -6,7 +6,7 @@ module BulkImports attr_reader :data def initialize(data: nil, page_info: {}) - @data = Array.wrap(data) + @data = data.is_a?(Enumerator) ? data : Array.wrap(data) @page_info = page_info end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index 35b77240ea7..bc7fc14b5a0 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -29,9 +29,13 @@ module BulkImports pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, stage: 1 }, + boards: { + pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, + stage: 2 + }, finisher: { pipeline: BulkImports::Groups::Pipelines::EntityFinisher, - stage: 2 + stage: 3 } }.freeze diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb index 43ceed9519b..f270f7984da 100644 --- a/lib/csv_builder.rb +++ b/lib/csv_builder.rb @@ -16,7 +16,7 @@ class CsvBuilder DEFAULT_ORDER_BY = 'id' DEFAULT_BATCH_SIZE = 1000 - PREFIX_REGEX = /^[=\+\-@;]/.freeze + PREFIX_REGEX = /\A[=\+\-@;]/.freeze attr_reader :rows_written diff --git a/lib/feature.rb b/lib/feature.rb index 87abd2689d0..453ecc8255a 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -18,6 +18,10 @@ class Feature superclass.table_name = 'feature_gates' end + # To enable EE overrides + class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore + end + InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException class << self @@ -167,7 +171,8 @@ class Feature ActiveSupportCacheStoreAdapter.new( active_record_adapter, l2_cache_backend, - expires_in: 1.hour) + expires_in: 1.hour, + write_through: true) # Thread-local L1 cache: use a short timeout since we don't have a # way to expire this cache all at once diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb deleted file mode 100644 index 431f1169a86..00000000000 --- a/lib/feature/active_support_cache_store_adapter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -# rubocop:disable Gitlab/NamespacedClass -# This class was already nested this way before moving to a separate file -class Feature - class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore - # This patch represents https://github.com/jnunemaker/flipper/pull/512. In - # Flipper 0.21.0 and later, we can remove this and just pass `write_through: - # true` to the constructor in `Feature.build_flipper_instance`. - - extend ::Gitlab::Utils::Override - - override :enable - def enable(feature, gate, thing) - result = @adapter.enable(feature, gate, thing) - @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) - result - end - - override :disable - def disable(feature, gate, thing) - result = @adapter.disable(feature, gate, thing) - @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) - result - end - - override :remove - def remove(feature) - result = @adapter.remove(feature) - @cache.delete(FeaturesKey) - @cache.write(key_for(feature.key), {}, @write_options) - result - end - end -end -# rubocop:disable Gitlab/NamespacedClass diff --git a/lib/flowdock/git.rb b/lib/flowdock/git.rb index 539fd66a510..897ee647d87 100644 --- a/lib/flowdock/git.rb +++ b/lib/flowdock/git.rb @@ -34,7 +34,7 @@ module Flowdock # Send git push notification to Flowdock def post messages.each do |message| - Flowdock::Client.new(flow_token: @token).post_to_thread(message) + ::Flowdock::Client.new(flow_token: @token).post_to_thread(message) end end diff --git a/lib/generators/gitlab/usage_metric/USAGE b/lib/generators/gitlab/usage_metric/USAGE new file mode 100644 index 00000000000..3a2166c3bb1 --- /dev/null +++ b/lib/generators/gitlab/usage_metric/USAGE @@ -0,0 +1,9 @@ +Description: + Creates a stub instrumentation for a Service Ping metric + +Example: + rails generate gitlab:usage_metric CountIssues --type database + + This will create: + lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb + spec/lib/gitlab/usage/metrics/instrumentations/count_issues_metric_spec.rb diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb b/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template index 9c92f2e9595..603b6f3bc8a 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb +++ b/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template @@ -4,8 +4,9 @@ module Gitlab module Usage module Metrics module Instrumentations - class CountUsersUsingApproveQuickActionMetric < RedisHLLMetric - event_names :i_quickactions_approve + class <%= class_name %>Metric < <%= metric_superclass %>Metric + def value + end end end end diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template new file mode 100644 index 00000000000..e984daee0a4 --- /dev/null +++ b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric do + it_behaves_like 'a correct instrumented metric value', {}, 1 +end diff --git a/lib/generators/gitlab/usage_metric/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric/usage_metric_generator.rb new file mode 100644 index 00000000000..f7125fdc911 --- /dev/null +++ b/lib/generators/gitlab/usage_metric/usage_metric_generator.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + class UsageMetricGenerator < Rails::Generators::Base + CE_DIR = 'lib/gitlab/usage/metrics/instrumentations' + EE_DIR = 'ee/lib/ee/gitlab/usage/metrics/instrumentations' + SPEC_CE_DIR = 'spec/lib/gitlab/usage/metrics/instrumentations' + SPEC_EE_DIR = 'ee/spec/lib/ee/gitlab/usage/metrics/instrumentations' + + ALLOWED_SUPERCLASSES = { + generic: 'Generic', + database: 'Database', + redis_hll: 'RedisHLL' + }.freeze + + source_root File.expand_path('templates', __dir__) + + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if instrumentation is for EE' + class_option :type, type: :string, desc: "Metric type, must be one of: #{ALLOWED_SUPERCLASSES.keys.join(', ')}" + + argument :class_name, type: :string, desc: 'Instrumentation class name, e.g.: CountIssues' + + def create_class_files + validate! + + template "instrumentation_class.rb.template", file_path + template "instrumentation_class_spec.rb.template", spec_file_path + end + + private + + def validate! + raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present? + raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil? + end + + def ee? + options[:ee] + end + + def type + options[:type] + end + + def file_path + dir = ee? ? EE_DIR : CE_DIR + + File.join(dir, file_name) + end + + def spec_file_path + dir = ee? ? SPEC_EE_DIR : SPEC_CE_DIR + + File.join(dir, spec_file_name) + end + + def file_name + "#{class_name.underscore}_metric.rb" + end + + def spec_file_name + "#{class_name.underscore}_metric_spec.rb" + end + + def metric_superclass + ALLOWED_SUPERCLASSES[type.to_sym] + end + end +end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 86bb2f662e5..d93d7acbaad 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -39,17 +39,14 @@ module Gitlab end end - COM_URL = 'https://gitlab.com' - STAGING_COM_URL = 'https://staging.gitlab.com' APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze - SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze HTTP_PROXY_ENV_VARS = %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY).freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com - Gitlab.config.gitlab.url == COM_URL || gl_subdomain? + Gitlab.config.gitlab.url == Gitlab::Saas.com_url || gl_subdomain? end def self.com @@ -57,7 +54,7 @@ module Gitlab end def self.staging? - Gitlab.config.gitlab.url == STAGING_COM_URL + Gitlab.config.gitlab.url == Gitlab::Saas.staging_com_url end def self.canary? @@ -73,11 +70,11 @@ module Gitlab end def self.org? - Gitlab.config.gitlab.url == 'https://dev.gitlab.org' + Gitlab.config.gitlab.url == Gitlab::Saas.dev_url end def self.gl_subdomain? - SUBDOMAIN_REGEX === Gitlab.config.gitlab.url + Gitlab::Saas.subdomain_regex === Gitlab.config.gitlab.url end def self.dev_env_org_or_com? diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 601f2175cfc..760f1352256 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -44,6 +44,10 @@ module Gitlab current.include?(Labkit::Context.log_key(attribute_name)) end + def self.current_context_attribute(attribute_name) + Labkit::Context.current&.get_attribute(attribute_name) + end + def initialize(**args) unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4489fc9f3b2..36f58d43a77 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -156,15 +156,16 @@ module Gitlab underscored_service = matched_login['service'].underscore - if Integration.available_services_names.include?(underscored_service) - # We treat underscored_service as a trusted input because it is included - # in the Integration.available_services_names allowlist. - service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend + return unless Integration.available_services_names.include?(underscored_service) - if service && service.activated? && service.valid_token?(password) - Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) - end - end + # We treat underscored_service as a trusted input because it is included + # in the Integration.available_services_names allowlist. + accessor = Project.integration_association_name(underscored_service) + service = project.public_send(accessor) # rubocop:disable GitlabSecurity/PublicSend + + return unless service && service.activated? && service.valid_token?(password) + + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) end def user_with_password_for_git(login, password) @@ -371,7 +372,9 @@ module Gitlab end def find_build_by_token(token) - ::Ci::AuthJobFinder.new(token: token).execute + ::Gitlab::Database::LoadBalancing::Session.current.use_primary do + ::Ci::AuthJobFinder.new(token: token).execute + end end def user_auth_attempt!(user, success:) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 523452d1074..1c5ded2e8ed 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -208,7 +208,7 @@ module Gitlab def build_new_user(skip_confirmation: true) user_params = user_attributes.merge(skip_confirmation: skip_confirmation) - Users::BuildService.new(nil, user_params).execute(skip_authorization: true) + Users::AuthorizedBuildService.new(nil, user_params).execute end def user_attributes diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb new file mode 100644 index 00000000000..cb9b0e88ef4 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The migration is used to cleanup orphaned lfs_objects_projects in order to + # introduce valid foreign keys to this table + class CleanupOrphanedLfsObjectsProjects + # A model to access lfs_objects_projects table in migrations + class LfsObjectsProject < ActiveRecord::Base + self.table_name = 'lfs_objects_projects' + + include ::EachBatch + + belongs_to :lfs_object + belongs_to :project + end + + # A model to access lfs_objects table in migrations + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + # A model to access projects table in migrations + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + SUB_BATCH_SIZE = 5000 + CLEAR_CACHE_DELAY = 1.minute + + def perform(start_id, end_id) + cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + cleanup_lfs_objects_projects_without_project(start_id, end_id) + end + + private + + def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects| + projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id)) + + if projects.present? + ProjectCacheWorker.bulk_perform_in_with_contexts( + CLEAR_CACHE_DELAY, + projects, + arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] }, + context_proc: ->(project) { { project: project } } + ) + end + + lfs_objects_projects_without_lfs_objects.delete_all + end + end + + def cleanup_lfs_objects_projects_without_project(start_id, end_id) + each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects| + lfs_objects_projects_without_projects.delete_all + end + end + + def each_record_without_association(start_id, end_id, association, table_name) + batch = LfsObjectsProject.where(id: start_id..end_id) + + batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)')).first + + lfs_objects_without_association = + LfsObjectsProject + .unscoped + .left_outer_joins(association) + .where(id: (first..last), table_name => { id: nil }) + + yield lfs_objects_without_association + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb new file mode 100644 index 00000000000..9a88eb8ea06 --- /dev/null +++ b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + BATCH_SIZE = 1000 + + # This background migration disables container expiration policies connected + # to a project that has no container repositories + class DisableExpirationPoliciesLinkedToNoContainerImages + # rubocop: disable Style/Documentation + class ContainerExpirationPolicy < ActiveRecord::Base + include EachBatch + + self.table_name = 'container_expiration_policies' + end + # rubocop: enable Style/Documentation + + def perform(from_id, to_id) + ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch| + sql = <<-SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql}) + UPDATE container_expiration_policies + SET enabled = FALSE + FROM batched_relation + WHERE container_expiration_policies.project_id = batched_relation.project_id + AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id) + SQL + execute(sql) + end + end + + private + + def execute(sql) + ActiveRecord::Base + .connection + .execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 888a12f2330..a00d291245c 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,6 +58,13 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + + logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', + finding_ids: mappings.keys.pluck(:id)) + + mark_job_as_succeeded(start_id, end_id) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end private @@ -76,4 +83,15 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid CalculateFindingUUID.call(name) end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'RecalculateVulnerabilitiesOccurrencesUuid', + arguments + ) + end end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb new file mode 100644 index 00000000000..bba1ca26b35 --- /dev/null +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeBasedOnUrl + # rubocop: disable Gitlab/NamespacedClass + class JiraTrackerData < ActiveRecord::Base + self.table_name = "jira_tracker_data" + self.inheritance_column = :_type_disabled + + include ::Integrations::BaseDataFields + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end + # rubocop: enable Gitlab/NamespacedClass + + # https://rubular.com/r/uwgK7k9KH23efa + JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix.freeze + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + trackers_data = JiraTrackerData + .where(deployment_type: 'unknown') + .where(id: start_id..end_id) + + cloud, server = trackers_data.partition { |tracker_data| tracker_data.url.match?(JIRA_CLOUD_REGEX) } + + cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 2 } + end + + server_mapppings = server.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 1 } + end + + mappings = cloud_mappings.merge(server_mapppings) + + ::Gitlab::Database::BulkUpdate.execute(%i[deployment_type], mappings) + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/lib/gitlab/cache.rb b/lib/gitlab/cache.rb index 90a0c38ff7b..433614a3007 100644 --- a/lib/gitlab/cache.rb +++ b/lib/gitlab/cache.rb @@ -13,6 +13,13 @@ module Gitlab end end end + + # Hook for EE + def delete(key) + Rails.cache.delete(key) + end end end end + +Gitlab::Cache.prepend_mod diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index ec94991157a..86441973941 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -113,6 +113,17 @@ module Gitlab end end + # Returns the values of the given set. + # + # raw_key - The key of the set to check. + def self.values_from_set(raw_key) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.smembers(key) + end + end + # Sets multiple keys to given values. # # mapping - A Hash mapping the cache keys to their values. diff --git a/lib/gitlab/checks/base_bulk_checker.rb b/lib/gitlab/checks/base_bulk_checker.rb new file mode 100644 index 00000000000..46a68fdf485 --- /dev/null +++ b/lib/gitlab/checks/base_bulk_checker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseBulkChecker < BaseChecker + attr_reader :changes_access + delegate(*ChangesAccess::ATTRIBUTES, to: :changes_access) + + def initialize(changes_access) + @changes_access = changes_access + end + + def validate! + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index 68873610408..2b0af7dc4f6 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -5,39 +5,16 @@ module Gitlab class BaseChecker include Gitlab::Utils::StrongMemoize - attr_reader :change_access - delegate(*ChangeAccess::ATTRIBUTES, to: :change_access) - - def initialize(change_access) - @change_access = change_access - end - def validate! raise NotImplementedError end private - def creation? - Gitlab::Git.blank_ref?(oldrev) - end - - def deletion? - Gitlab::Git.blank_ref?(newrev) - end - - def update? - !creation? && !deletion? - end - def updated_from_web? protocol == 'web' end - def tag_exists? - project.repository.tag_exists?(tag_name) - end - def validate_once(resource) Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do yield(resource) diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb new file mode 100644 index 00000000000..f93902055c9 --- /dev/null +++ b/lib/gitlab/checks/base_single_checker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseSingleChecker < BaseChecker + attr_reader :change_access + delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) + + def initialize(change_access) + @change_access = change_access + end + + private + + def creation? + Gitlab::Git.blank_ref?(oldrev) + end + + def deletion? + Gitlab::Git.blank_ref?(newrev) + end + + def update? + !creation? && !deletion? + end + + def tag_exists? + project.repository.tag_exists?(tag_name) + end + end + end +end + +Gitlab::Checks::BaseSingleChecker.prepend_mod_with('Gitlab::Checks::BaseSingleChecker') diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index a8287a97cc3..a2d74d36b58 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class BranchCheck < BaseChecker + class BranchCheck < BaseSingleChecker ERROR_MESSAGES = { delete_default_branch: 'The default branch of a project cannot be deleted.', force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb new file mode 100644 index 00000000000..4e8b293a3e6 --- /dev/null +++ b/lib/gitlab/checks/changes_access.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class ChangesAccess + ATTRIBUTES = %i[user_access project protocol changes logger].freeze + + attr_reader(*ATTRIBUTES) + + def initialize( + changes, user_access:, project:, protocol:, logger: + ) + @changes = changes + @user_access = user_access + @project = project + @protocol = protocol + @logger = logger + end + + def validate! + return if changes.empty? + + single_access_checks! + + logger.log_timed("Running checks for #{changes.length} changes") do + bulk_access_checks! + end + + true + end + + protected + + def single_access_checks! + # Iterate over all changes to find if user allowed all of them to be applied + changes.each do |change| + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + Checks::SingleChangeAccess.new( + change, + user_access: user_access, + project: project, + protocol: protocol, + logger: logger + ).validate! + end + end + + def bulk_access_checks! + Gitlab::Checks::LfsCheck.new(self).validate! + end + end + end +end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index a05181ab58e..d8f5cec8a4a 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class DiffCheck < BaseChecker + class DiffCheck < BaseSingleChecker include Gitlab::Utils::StrongMemoize LOG_MESSAGES = { diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 38f0b82c8b4..51013b69755 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class LfsCheck < BaseChecker + class LfsCheck < BaseBulkChecker LOG_MESSAGE = 'Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...' ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' @@ -12,11 +12,10 @@ module Gitlab return unless Feature.enabled?(:lfs_check, default_enabled: true) return unless project.lfs_enabled? - return if skip_lfs_integrity_check - return if deletion? logger.log_timed(LOG_MESSAGE) do - lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) + newrevs = changes.map { |change| change[:newrev] } + lfs_check = Checks::LfsIntegrity.new(project, newrevs, logger.time_left) if lfs_check.objects_missing? raise GitAccess::ForbiddenError, ERROR_MESSAGE diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 78952db7a3e..845fb2da925 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -3,16 +3,19 @@ module Gitlab module Checks class LfsIntegrity - def initialize(project, newrev, time_left) + def initialize(project, newrevs, time_left) @project = project - @newrev = newrev + @newrevs = newrevs @time_left = time_left end def objects_missing? - return false unless @newrev && @project.lfs_enabled? + return false unless @project.lfs_enabled? - new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev) + newrevs = @newrevs.reject { |rev| rev.blank? || Gitlab::Git.blank_ref?(rev) } + return if newrevs.blank? + + new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, newrevs) .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left) return false unless new_lfs_pointers.present? diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 2635ad04770..e37cbc0442b 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -3,22 +3,74 @@ module Gitlab module Checks class MatchingMergeRequest + TOTAL_METRIC = :gitlab_merge_request_match_total + STALE_METRIC = :gitlab_merge_request_match_stale_secondary + def initialize(newrev, branch_name, project) @newrev = newrev @branch_name = branch_name @project = project end - # rubocop: disable CodeReuse/ActiveRecord def match? + if ::Gitlab::Database::LoadBalancing.enable? + # When a user merges a merge request, the following sequence happens: + # + # 1. Sidekiq: MergeService runs and updates the merge request in a locked state. + # 2. Gitaly: The UserMergeBranch RPC runs. + # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook. + # 4. Rails: This hook makes an API request to /api/v4/internal/allowed. + # 5. Rails: This API check does a SQL query for locked merge + # requests with a matching SHA. + # + # Since steps 1 and 5 will happen on different database + # sessions, replication lag could erroneously cause step 5 to + # report no matching merge requests. To avoid this, we check + # the write location to ensure the replica can make this query. + track_session_metrics do + ::Gitlab::Database::LoadBalancing::Sticking.select_valid_host(:project, @project.id) + end + end + + # rubocop: disable CodeReuse/ActiveRecord @project.merge_requests .with_state(:locked) .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) .exists? + # rubocop: enable CodeReuse/ActiveRecord + end + + private + + def track_session_metrics + before = ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + + yield + + after = ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + + increment_attempt_count + + if !before && after + increment_stale_secondary_count + end + end + + def increment_attempt_count + total_counter.increment + end + + def increment_stale_secondary_count + stale_counter.increment + end + + def total_counter + @total_counter ||= ::Gitlab::Metrics.counter(TOTAL_METRIC, 'Total number of merge request match attempts') + end + + def stale_counter + @stale_counter ||= ::Gitlab::Metrics.counter(STALE_METRIC, 'Total number of merge request match attempts with lagging secondary') end - # rubocop: enable CodeReuse/ActiveRecord end end end - -Gitlab::Checks::MatchingMergeRequest.prepend_mod_with('Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index 47aa25aae4c..50002e00a77 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class PushCheck < BaseChecker + class PushCheck < BaseSingleChecker def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? diff --git a/lib/gitlab/checks/push_file_count_check.rb b/lib/gitlab/checks/push_file_count_check.rb index 288a7e0d41a..707d4cfbcbe 100644 --- a/lib/gitlab/checks/push_file_count_check.rb +++ b/lib/gitlab/checks/push_file_count_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class PushFileCountCheck < BaseChecker + class PushFileCountCheck < BaseSingleChecker attr_reader :repository, :newrev, :limit, :logger LOG_MESSAGES = { diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/single_change_access.rb index a2c3de3e775..280b2dd25e2 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -2,23 +2,22 @@ module Gitlab module Checks - class ChangeAccess + class SingleChangeAccess ATTRIBUTES = %i[user_access project skip_authorization - skip_lfs_integrity_check protocol oldrev newrev ref + protocol oldrev newrev ref branch_name tag_name logger commits].freeze attr_reader(*ATTRIBUTES) def initialize( change, user_access:, project:, - skip_lfs_integrity_check: false, protocol:, logger: + protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol @logger = logger @@ -44,7 +43,6 @@ module Gitlab Gitlab::Checks::PushCheck.new(self).validate! Gitlab::Checks::BranchCheck.new(self).validate! Gitlab::Checks::TagCheck.new(self).validate! - Gitlab::Checks::LfsCheck.new(self).validate! end def commits_check @@ -54,4 +52,4 @@ module Gitlab end end -Gitlab::Checks::ChangeAccess.prepend_mod_with('Gitlab::Checks::ChangeAccess') +Gitlab::Checks::SingleChangeAccess.prepend_mod_with('Gitlab::Checks::SingleChangeAccess') diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb index d5efbfcc5bc..43168600ec9 100644 --- a/lib/gitlab/checks/snippet_check.rb +++ b/lib/gitlab/checks/snippet_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class SnippetCheck < BaseChecker + class SnippetCheck < BaseSingleChecker ERROR_MESSAGES = { create_delete_branch: 'You can not create or delete branches.' }.freeze diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index a47e55cb160..a45db85301a 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -2,7 +2,7 @@ module Gitlab module Checks - class TagCheck < BaseChecker + class TagCheck < BaseSingleChecker ERROR_MESSAGES = { change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index b1dee0e1ecc..466706384c0 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -77,7 +77,7 @@ module Gitlab end def set_section_duration(duration) - @section_duration = Time.at(duration.to_i).strftime('%M:%S') + @section_duration = Time.at(duration.to_i).utc.strftime('%M:%S') end def flush_current_segment! diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 7589fa5ff8b..96702420e9d 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -24,26 +24,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text @status ? ("%.2f%%" % @status) : 'unknown' end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width @status ? 54 : 58 end diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index 8430b01fc9a..c39f96e4a34 100644 --- a/lib/gitlab/ci/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -28,26 +28,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text STATUS_RENAME[@status.to_s] || @status.to_s end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width 54 end diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 0580dad72ba..d514a8577bd 100644 --- a/lib/gitlab/ci/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -8,6 +8,7 @@ module Gitlab::Ci class Template MAX_KEY_TEXT_SIZE = 64 MAX_KEY_WIDTH = 512 + DEFAULT_KEY_WIDTH = 62 def initialize(badge) @entity = badge.entity @@ -15,7 +16,11 @@ module Gitlab::Ci end def key_text - raise NotImplementedError + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE + @key_text + else + @entity.to_s + end end def value_text @@ -23,7 +28,11 @@ module Gitlab::Ci end def key_width - raise NotImplementedError + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) + @key_width + else + DEFAULT_KEY_WIDTH + end end def value_width diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index e6ef12975c2..b98d1d7b330 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -7,6 +7,11 @@ class Gitlab::Ci::Build::AutoRetry scheduler_failure: 2 }.freeze + RETRY_OVERRIDES = { + ci_quota_exceeded: 0, + no_matching_runner: 0 + }.freeze + def initialize(build) @build = build end @@ -19,13 +24,18 @@ class Gitlab::Ci::Build::AutoRetry private + delegate :failure_reason, to: :@build + def within_max_retry_limit? max_allowed_retries > 0 && max_allowed_retries > @build.retries_count end def max_allowed_retries strong_memoize(:max_allowed_retries) do - options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0) + RETRY_OVERRIDES[failure_reason.to_sym] || + options_retry_max || + DEFAULT_RETRIES[failure_reason.to_sym] || + 0 end end @@ -38,7 +48,7 @@ class Gitlab::Ci::Build::AutoRetry end def retry_on_reason_or_always? - options_retry_when.include?(@build.failure_reason.to_s) || + options_retry_when.include?(failure_reason.to_s) || options_retry_when.include?('always') end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index 29dc48c7b42..f1b67635c08 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -35,14 +35,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: @config, - artifacts: true, - optional: false } - else - { name: @config, - artifacts: true } - end + { name: @config, + artifacts: true, + optional: false } end end @@ -66,14 +61,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: job, - artifacts: artifacts || artifacts.nil?, - optional: !!optional } - else - { name: job, - artifacts: artifacts || artifacts.nil? } - end + { name: job, + artifacts: artifacts || artifacts.nil?, + optional: !!optional } end end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 947b6787aa0..79dfb0eec1d 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -98,7 +98,6 @@ module Gitlab def validate_against_warnings # If rules are valid format and workflow rules are not specified return unless rules_value - return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? last_rule = rules_value.last diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f2fd8ac7fd9..4db25fb0930 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast secret_detection dependency_scanning container_scanning - dast performance browser_performance load_performance license_management license_scanning metrics lsif + dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications requirements coverage_fuzzing api_fuzzing].freeze @@ -36,7 +36,6 @@ module Gitlab validates :performance, array_of_strings_or_string: true validates :browser_performance, array_of_strings_or_string: true validates :load_performance, array_of_strings_or_string: true - validates :license_management, array_of_strings_or_string: true validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true @@ -44,7 +43,7 @@ module Gitlab validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true - validates :cluster_applications, array_of_strings_or_string: true + validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index a8f78b62d8d..e6ff33d6f79 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -28,11 +28,6 @@ module Gitlab end end - def matching? - super && - Feature.enabled?(:ci_dynamic_child_pipeline, project, default_enabled: true) - end - private def project diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index c4b4a7a0a73..47441fa3818 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -6,7 +6,7 @@ module Gitlab module External module File class Template < Base - attr_reader :location, :project + attr_reader :location SUFFIX = '.gitlab-ci.yml' @@ -41,7 +41,7 @@ module Gitlab end def fetch_template_content - Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index efd48a9b29f..bc03658aab8 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,6 +6,10 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' VALID_SYNTAX_SAMPLE_CRON = '* * * * *' + def self.parse_natural(expression, cron_timezone = 'UTC') + new(Fugit::Nat.parse(expression)&.original, cron_timezone) + end + def initialize(cron, cron_timezone = 'UTC') @cron = cron @cron_timezone = timezone_name(cron_timezone) @@ -27,6 +31,10 @@ module Gitlab try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? end + def match?(time) + cron_line.match?(time) + end + private def timezone_name(timezone) diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c8e4d9ed763..fe69a170404 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,14 +18,6 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - def self.merge_base_pipeline_for_metrics_comparison?(project) - Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) - end - - def self.raise_job_rules_without_workflow_rules_warning? - ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -33,10 +25,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.trace_overwrite? - ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) - end - def self.accept_trace?(project) ::Feature.enabled?(:ci_enable_live_trace, project) && ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) @@ -53,10 +41,6 @@ module Gitlab def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end - - def self.background_pipeline_retry_endpoint?(project) - ::Feature.enabled?(:background_pipeline_retry_endpoint, project) - end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0b94debb24e..3fb86b8b3e8 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -54,6 +54,7 @@ module Gitlab user_login: user&.username, user_email: user&.email, pipeline_id: build.pipeline.id.to_s, + pipeline_source: build.pipeline.source.to_s, job_id: build.id.to_s, ref: source_ref, ref_type: ref_type, diff --git a/lib/gitlab/ci/matching/build_matcher.rb b/lib/gitlab/ci/matching/build_matcher.rb new file mode 100644 index 00000000000..dff7d9141d9 --- /dev/null +++ b/lib/gitlab/ci/matching/build_matcher.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + class BuildMatcher + ATTRIBUTES = %i[ + protected + tag_list + build_ids + project + ].freeze + + attr_reader(*ATTRIBUTES) + alias_method :protected?, :protected + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def has_tags? + tag_list.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/matching/runner_matcher.rb b/lib/gitlab/ci/matching/runner_matcher.rb new file mode 100644 index 00000000000..63642674936 --- /dev/null +++ b/lib/gitlab/ci/matching/runner_matcher.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + ### + # This class is used to check if a build can be picked by a runner: + # + # runner = Ci::Runner.find(id) + # build = Ci::Build.find(id) + # runner.runner_matcher.matches?(build.build_matcher) + # + # There are also class level methods to build matchers: + # + # `project.builds.build_matchers(project)` returns a distinct collection + # of build matchers. + # `Ci::Runner.runner_matchers` returns a distinct collection of runner matchers. + # + class RunnerMatcher + ATTRIBUTES = %i[ + runner_type + public_projects_minutes_cost_factor + private_projects_minutes_cost_factor + run_untagged + access_level + tag_list + ].freeze + + attr_reader(*ATTRIBUTES) + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def matches?(build_matcher) + ensure_build_matcher_instance!(build_matcher) + return false if ref_protected? && !build_matcher.protected? + + accepting_tags?(build_matcher) + end + + def instance_type? + runner_type.to_sym == :instance_type + end + + private + + def ref_protected? + access_level.to_sym == :ref_protected + end + + def accepting_tags?(build_matcher) + (run_untagged || build_matcher.has_tags?) && (build_matcher.tag_list - tag_list).empty? + end + + def ensure_build_matcher_instance!(build_matcher) + return if build_matcher.is_a?(Matching::BuildMatcher) + + raise ArgumentError, 'only Gitlab::Ci::Matching::BuildMatcher are allowed' + end + end + end + end +end + +Gitlab::Ci::Matching::RunnerMatcher.prepend_mod_with('Gitlab::Ci::Matching::RunnerMatcher') diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index ca7fbde6713..364ae66844e 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -69,6 +69,7 @@ module Gitlab elsif data.key?('error') status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR system_output = data['error'] + attachment = attachment_path(data['system_out']) elsif data.key?('skipped') status = ::Gitlab::Ci::Reports::TestCase::STATUS_SKIPPED system_output = data['skipped'] diff --git a/lib/gitlab/ci/pipeline/chain/validate/after_config.rb b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb new file mode 100644 index 00000000000..c3db00b4fb2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class AfterConfig < Chain::Base + include Chain::Helpers + + def perform! + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 539b44513f0..27bb7fdc05a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -12,12 +12,9 @@ module Gitlab DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 ACCEPTED_STATUS = 200 - DOT_COM_REJECTED_STATUS = 406 - GENERAL_REJECTED_STATUS = (400..499).freeze + REJECTED_STATUS = 406 def perform! - return unless enabled? - pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' @@ -32,24 +29,17 @@ module Gitlab private - def enabled? - return true unless Gitlab.com? - - ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) - end - def validate_external return true unless validation_service_url # 200 - accepted - # 406 - not accepted on GitLab.com - # 4XX - not accepted for other installations + # 406 - rejected # everything else - accepted and logged response_code = validate_service_request.code case response_code when ACCEPTED_STATUS true - when rejected_status + when REJECTED_STATUS false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" @@ -60,14 +50,6 @@ module Gitlab true end - def rejected_status - if Gitlab.com? - DOT_COM_REJECTED_STATUS - else - GENERAL_REJECTED_STATUS - end - end - def validate_service_request headers = { 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, @@ -107,7 +89,9 @@ module Gitlab id: current_user.id, username: current_user.username, email: current_user.email, - created_at: current_user.created_at&.iso8601 + created_at: current_user.created_at&.iso8601, + current_sign_in_ip: current_user.current_sign_in_ip, + last_sign_in_ip: current_user.last_sign_in_ip }, pipeline: { sha: pipeline.sha, diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb index 7befc126ca9..31ddf2c4241 100644 --- a/lib/gitlab/ci/pipeline/preloader.rb +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -20,6 +20,7 @@ module Gitlab preloader.preload_ref_commits preloader.preload_pipeline_warnings preloader.preload_stages_warnings + preloader.preload_persisted_environments end end end @@ -54,6 +55,13 @@ module Gitlab def preload_stages_warnings @pipeline.stages.each { |stage| stage.number_of_warnings } end + + # This batch loads the associated environments of multiple actions (builds) + # that can't use `preload` due to the indirect relationship. + def preload_persisted_environments + @pipeline.scheduled_actions.each { |action| action.persisted_environment } + @pipeline.manual_actions.each { |action| action.persisted_environment } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 39dee7750d6..299b27a5f13 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -146,7 +146,7 @@ module Gitlab end @needs_attributes.flat_map do |need| - next if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) && need[:optional] + next if need[:optional] result = @previous_stages.any? do |stage| stage.seeds_names.include?(need[:name]) diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 46e4373ec85..859aeb35f26 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -20,6 +20,8 @@ module Gitlab :build_can_pick, :build_not_pick, :build_not_pending, + :build_queue_push, + :build_queue_pop, :build_temporary_locked, :build_conflict_lock, :build_conflict_exception, @@ -31,7 +33,9 @@ module Gitlab :queue_replication_lag, :runner_pre_assign_checks_failed, :runner_pre_assign_checks_success, - :runner_queue_tick + :runner_queue_tick, + :shared_runner_build_new, + :shared_runner_build_done ].to_set.freeze QUEUE_DEPTH_HISTOGRAMS = [ @@ -77,11 +81,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def increment_queue_operation(operation) - if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) - raise ArgumentError, "unknown queue operation: #{operation}" - end - - self.class.queue_operations_total.increment(operation: operation) + self.class.increment_queue_operation(operation) end def observe_queue_depth(queue, size) @@ -121,6 +121,14 @@ module Gitlab result end + def self.increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + queue_operations_total.increment(operation: operation) + end + def self.observe_active_runners(runners_proc) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 239fc3b15e7..287a03cefe2 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -8,6 +8,7 @@ module Gitlab DEFAULT_MAX_TESTS = 100 DEFAULT_MIN_TESTS = 10 + TestSummary = Struct.new(:new_failures, :existing_failures, :resolved_failures, :new_errors, :existing_errors, :resolved_errors, keyword_init: true) attr_reader :name, :base_suite, :head_suite @@ -90,7 +91,7 @@ module Gitlab def limited_tests strong_memoize(:limited_tests) do # rubocop: disable CodeReuse/ActiveRecord - OpenStruct.new( + TestSummary.new( new_failures: new_failures.take(max_tests), existing_failures: existing_failures.take(max_tests(new_failures)), resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)), diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index cbd72f54ff4..66f51f63585 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -30,7 +30,8 @@ module Gitlab reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', - ci_quota_exceeded: 'no more CI minutes available' + ci_quota_exceeded: 'no more CI minutes available', + no_matching_runner: 'no matching runner available' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a13f2046291..5680950bba8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -11,7 +11,7 @@ # * test: TEST_DISABLED # * code_quality: CODE_QUALITY_DISABLED # * license_management: LICENSE_MANAGEMENT_DISABLED -# * performance: PERFORMANCE_DISABLED +# * browser_performance: BROWSER_PERFORMANCE_DISABLED # * load_performance: LOAD_PERFORMANCE_DISABLED # * sast: SAST_DISABLED # * secret_detection: SECRET_DETECTION_DISABLED diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 4dc88418671..07d0de5f9e5 100644 --- a/lib/gitlab/ci/templates/Getting-started.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -16,7 +16,7 @@ build-job: # This job runs in the build stage, which runs first. stage: build script: - echo "Compiling the code..." - - echo "Compile complete. + - echo "Compile complete." unit-test-job: # This job runs in the test stage. stage: test # It only starts when the job in the build stage completes successfully. diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 01907ef9e2e..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html -performance: +browser_performance: stage: performance image: docker:19.03.12 allow_failure: true @@ -72,6 +72,6 @@ performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 5216a46745c..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -72,6 +72,6 @@ browser_performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index abcb347b146..cf99d722e4d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0' variables: - DOCKER_TLS_CERTDIR: "" + DOCKER_TLS_CERTDIR: '' services: - - name: "docker:20.10.6-dind" + - name: 'docker:20.10.6-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b29342216fc..48e877684f6 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7ad5a9e2bba..00fcfa64a18 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 29edada4041..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml new file mode 100644 index 00000000000..6af79728dc8 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -0,0 +1,335 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + SAST_EXCLUDED_ANALYZERS: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + SCAN_KUBERNETES_MANIFESTS: "false" + +sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + variables: + SEARCH_MAX_DEPTH: 4 + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + +.sast-analyzer: + extends: sast + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +bandit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + +brakeman-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.rb' + - '**/Gemfile' + +eslint-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +flawfinder-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.c' + - '**/*.cpp' + +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never + - if: $CI_COMMIT_BRANCH && + $SCAN_KUBERNETES_MANIFESTS == 'true' + +gosec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.go' + +.mobsf-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + +mobsf-android-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + +nodejs-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/package.json' + +phpcs-security-audit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.php' + +pmd-apex-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.cls' + +security-code-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.csproj' + - '**/*.vbproj' + +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +sobelow-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - 'mix.exs' + +spotbugs-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never + - if: $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml new file mode 100644 index 00000000000..d0595491400 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECRETS_ANALYZER_VERSION: "3" + SECRET_DETECTION_EXCLUDED_PATHS: "" + +.secret-analyzer: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + +secret_detection: + extends: .secret-analyzer + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi + - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - /analyzer run + - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 6f30fc2dcd5..ca63e942130 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,21 @@ +################################################################################ +# WARNING +################################################################################ +# +# This template is DEPRECATED and scheduled for removal in GitLab 15.0 +# See https://gitlab.com/gitlab-org/gitlab/-/issues/333610 for more context. +# +# To get started with a Cluster Management Project, we instead recommend +# using the updated project template: +# +# - Documentation: https://docs.gitlab.com/ee/user/clusters/management_project_template.html +# - Source code: https://gitlab.com/gitlab-org/project-templates/cluster-management/ +# +################################################################################ + apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.43.1" environment: name: production variables: @@ -9,11 +24,9 @@ apply: script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: - refs: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: - reports: - cluster_applications: gl-cluster-applications.json when: on_failure paths: - tiller.log diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 275364afae4..1bdaaeede43 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,6 +1,6 @@ # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/ruby/tags/ -image: "ruby:2.5" +image: ruby:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 90fad1550ff..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,279 +1,33 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables - -stages: - - build - - test - - deploy - - fuzz +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: + FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: "1.6" - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:5000 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:5000 - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 5000:5000 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 5001:5000 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8fa33026011..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: FUZZAPI_VERSION: "1" diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index c628e30b2c7..bd163f9db94 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,60 +1,44 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the DOCKER_IMAGE variable. If DOCKER_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `DOCKER_USER` and `DOCKER_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - CS_MAJOR_VERSION: 3 + CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 -.cs_common: - stage: test +container_scanning: image: "$CS_ANALYZER_IMAGE" + stage: test variables: - # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` - # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # for details + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting GIT_STRATEGY: none - # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true artifacts: reports: container_scanning: gl-container-scanning-report.json + paths: [gl-container-scanning-report.json] dependencies: [] - -container_scanning: - extends: .cs_common - variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" - CS_PROJECT: 'klar' - services: - - name: $CLAIR_DB_IMAGE - alias: clair-vulnerabilities-db - script: - - /analyzer run - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION =~ /^[0-3]$/ - -container_scanning_new: - extends: .cs_common - variables: - CS_PROJECT: 'container-scanning' script: - gtcs scan - artifacts: - paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION !~ /^[0-3]$/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 9d47537c0f0..2dbfb80b419 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,5 +1,8 @@ # Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables + variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index b40c4e982f7..9170e943e9d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -13,9 +13,8 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) -# List of variables available to configure the DAST API scanning tool: -# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: # Setting this variable affects all Security templates diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 7abecfb7e49..a2b112b8e9f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables stages: - build @@ -11,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index b6282da18a4..6834766da3d 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -13,12 +13,11 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" @@ -47,10 +46,13 @@ dast: $REVIEW_DISABLED && $DAST_WEBSITE == null && $DAST_API_SPECIFICATION == null when: never - - if: $CI_COMMIT_BRANCH && + - if: $CI_MERGE_REQUEST_IID && $CI_KUBERNETES_ACTIVE && $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_MERGE_REQUEST_IID && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) + - if: $CI_OPEN_MERGE_REQUESTS + when: never - if: $CI_COMMIT_BRANCH && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $DAST_API_SPECIFICATION + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 53d68c24d26..8df5ce79fe8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml deleted file mode 100644 index 87f78d0c887..00000000000 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624 -# Please, use License-Scanning.gitlab-ci.yml template instead - -include: - - template: License-Scanning.gitlab-ci.yml - -license_scanning: - before_script: - - | - echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template. - Please replace it with the License-Scanning.gitlab-ci.yml template instead. - For more details visit - https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 21e926ef275..870684c9f1d 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a8d45e80356..77ce813dd4f 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -1,340 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/SAST.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" - SAST_EXCLUDED_ANALYZERS: "" - SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" - SAST_ANALYZER_IMAGE_TAG: 2 - SCAN_KUBERNETES_MANIFESTS: "false" - -sast: - stage: test - artifacts: - reports: - sast: gl-sast-report.json - rules: - - when: never - variables: - SEARCH_MAX_DEPTH: 4 - script: - - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" - - exit 1 - -.sast-analyzer: - extends: sast - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - script: - - /analyzer run - -bandit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /bandit/ - exists: - - '**/*.py' - -brakeman-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /brakeman/ - exists: - - '**/*.rb' - - '**/Gemfile' - -eslint-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /eslint/ - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -flawfinder-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ - exists: - - '**/*.c' - - '**/*.cpp' - -kubesec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && - $SCAN_KUBERNETES_MANIFESTS == 'true' - -gosec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /gosec/ - exists: - - '**/*.go' - -.mobsf-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - -mobsf-android-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.apk' - - '**/AndroidManifest.xml' - -mobsf-ios-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.ipa' - - '**/*.xcodeproj/*' - -nodejs-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ - exists: - - '**/package.json' - -phpcs-security-audit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ - exists: - - '**/*.php' - -pmd-apex-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ - exists: - - '**/*.cls' - -security-code-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ - exists: - - '**/*.csproj' - - '**/*.vbproj' - -semgrep-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ - exists: - - '**/*.py' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -sobelow-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /sobelow/ - exists: - - 'mix.exs' - -spotbugs-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ - when: never - - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/AndroidManifest.xml' - when: never - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ - exists: - - '**/*.groovy' - - '**/*.java' - - '**/*.scala' - - '**/*.kt' +include: + template: Jobs/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index c255fb4707a..d4ea7165d0a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,45 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/Secret-Detection.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SECRETS_ANALYZER_VERSION: "3" - SECRET_DETECTION_EXCLUDED_PATHS: "" - - -.secret-analyzer: - stage: test - image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" - services: [] - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - artifacts: - reports: - secret_detection: gl-secret-detection-report.json - -secret_detection_default_branch: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH - script: - - /analyzer run - -secret_detection: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - script: - - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - - /analyzer run - - rm "$CI_COMMIT_SHA"_commit_list.txt +include: + template: Jobs/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index ac975fbbeab..d410c49b9a4 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -15,7 +15,6 @@ variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, - klar, clair-vulnerabilities-db, license-finder, dast, api-fuzzing @@ -78,6 +77,8 @@ brakeman: gosec: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -161,28 +162,6 @@ kubesec: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ -# -# Container Scanning jobs -# - -klar: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ - variables: - SECURE_BINARIES_ANALYZER_VERSION: "3" - -clair-vulnerabilities-db: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ - variables: - SECURE_BINARIES_IMAGE: arminc/clair-db - SECURE_BINARIES_ANALYZER_VERSION: latest # # Dependency Scanning jobs diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 6b9db1c2e0f..62b32d7c2db 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,61 +1,22 @@ -# Official image for Hashicorp's Terraform. It uses light image which is Alpine -# based as it is much lighter. -# -# Entrypoint is also needed as image by default set `terraform` binary as an -# entrypoint. -image: - name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - -# Default output file for Terraform plan -variables: - PLAN: plan.tfplan - JSON_PLAN_FILE: tfplan.json - -cache: - paths: - - .terraform - - .terraform.lock.hcl - -before_script: - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - - terraform --version - - terraform init +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: + - init - validate - build - - test - deploy +init: + extends: .init + validate: - stage: validate - script: - - terraform validate + extends: .validate -plan: - stage: build - script: - - terraform plan -out=$PLAN - - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" - artifacts: - paths: - - $PLAN - reports: - terraform: $JSON_PLAN_FILE +build: + extends: .build -# Separate apply job for manual launching Terraform as it can be destructive -# action. -apply: - stage: deploy - environment: - name: production - script: - - terraform apply -input=false $PLAN +deploy: + extends: .deploy dependencies: - - plan - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: manual + - build diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 404d4a4c6db..f0621165f8a 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - deploy - performance -performance: +browser_performance: stage: performance image: docker:git variables: diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 035ba52da84..536cf9bd8d8 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -1,22 +1,28 @@ -default: +publish: image: node:latest - - # Validate that the repository contains a package.json and extract a few values from it. - before_script: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ + changes: + - package.json + script: + # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - | - if [[ ! -f package.json ]]; then - echo "No package.json found! A package.json file is required to publish a package to GitLab's NPM registry." - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#creating-a-project' - exit 1 + if [[ ! -f .npmrc ]]; then + echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' + { + echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" + echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" + } >> .npmrc fi + - echo "Created the following .npmrc:"; cat .npmrc + + # Extract a few values from package.json - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") -# Validate that the package name is properly scoped to the project's root namespace. -# For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention -validate_package_scope: - stage: build - script: + # Validate that the package name is properly scoped to the project's root namespace. + # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - | if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" @@ -24,36 +30,12 @@ validate_package_scope: exit 1 fi -# If no .npmrc if included in the repo, generate a temporary one to use during the publish step -# that is configured to publish to GitLab's NPM registry -create_npmrc: - stage: build - script: + # Compare the version in package.json to all published versions. + # If the package.json version has not yet been published, run `npm publish`. - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#authenticating-with-a-ci-job-token' - - { - echo '@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN}' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}' - } >> .npmrc - - fi - artifacts: - paths: - - .npmrc - -# Publish the package. If the version in package.json has not yet been published, it will be -# published to GitLab's NPM registry. If the version already exists, the publish command -# will fail and the existing package will not be updated. -publish_package: - stage: deploy - script: - - | - { - npm publish && + if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + npm publish echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." - } + else + echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." + fi diff --git a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml deleted file mode 100644 index 536cf9bd8d8..00000000000 --- a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml +++ /dev/null @@ -1,41 +0,0 @@ -publish: - image: node:latest - stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ - changes: - - package.json - script: - # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' - { - echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" - echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" - } >> .npmrc - fi - - echo "Created the following .npmrc:"; cat .npmrc - - # Extract a few values from package.json - - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") - - # Validate that the package name is properly scoped to the project's root namespace. - # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - - | - if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then - echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention' - exit 1 - fi - - # Compare the version in package.json to all published versions. - # If the package.json version has not yet been published, run `npm publish`. - - | - if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then - npm publish - echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - else - echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." - fi diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c4757edf74e..84eb860a168 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -14,6 +14,8 @@ module Gitlab UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds + LOAD_BALANCING_STICKING_NAMESPACE = 'ci/build/trace' + ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) LockedError = Class.new(StandardError) @@ -296,25 +298,31 @@ module Gitlab read_trace_artifact(job) { job.job_artifacts_trace } end - ## - # Overridden in EE - # - def destroy_stream(job) + def destroy_stream(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end - ## - # Overriden in EE - # - def read_trace_artifact(job) + def read_trace_artifact(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end + def consistent_archived_trace?(build) + ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false) + end + def being_watched_cache_key "gitlab:ci:trace:#{job.id}:watched" end end end end - -::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 7c2e39b1e53..9f24ba99201 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -229,13 +229,8 @@ module Gitlab def next_chunk @chunks_cache[chunk_index] = begin - if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) - ::Ci::BuildTraceChunk - .safe_find_or_create_by(build: build, chunk_index: chunk_index) - else - ::Ci::BuildTraceChunk - .new(build: build, chunk_index: chunk_index) - end + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) end end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index ce9efbda7ea..fcd70634630 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -11,7 +11,6 @@ module Gitlab :streamed, # new trace data has been sent by a runner :chunked, # new trace chunk has been created :mutated, # trace has been mutated when removing secrets - :overwrite, # runner requested overwritting a build trace :accepted, # scheduled chunks for migration and responded with 202 :finalized, # all live build trace chunks have been persisted :discarded, # failed to persist live chunks before timeout diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e2a8af9c26b..ef9ba1b73c7 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -24,6 +24,10 @@ module Gitlab self end + def compact + Collection.new(select { |variable| !variable.value.nil? }) + end + def concat(resources) return self if resources.nil? @@ -64,11 +68,19 @@ module Gitlab end def expand_value(value, keep_undefined: false) - value.gsub(ExpandVariables::VARIABLES_REGEXP) do + value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match - result = @variables_by_key[match[1] || match[2]]&.value - result ||= match[0] if keep_undefined - result + if match[:key] + # we matched variable + if variable = @variables_by_key[match[:key]] + variable.value + elsif keep_undefined + match[0] + end + else + # we escape sequence + match[0] + end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 77da2c4cb91..0217e6129ca 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -7,6 +7,9 @@ module Gitlab class Item include Gitlab::Utils::StrongMemoize + VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/.freeze.freeze + VARIABLE_REF_CHARS = %w[$ %].freeze + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? @@ -34,9 +37,9 @@ module Gitlab strong_memoize(:depends_on) do next if raw - next unless ExpandVariables.possible_var_reference?(value) + next unless self.class.possible_var_reference?(value) - value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + value.scan(VARIABLES_REGEXP).filter_map(&:last) end end @@ -64,6 +67,12 @@ module Gitlab end end + def self.possible_var_reference?(value) + return unless value + + VARIABLE_REF_CHARS.any? { |symbol| value.include?(symbol) } + end + def to_s return to_runner_variable.to_s unless depends_on diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f96a6629849..15cc0c28296 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -111,6 +111,22 @@ module Gitlab @ci_config.variables_with_data end + def yaml_variables_for(job_name) + job = jobs[job_name] + + return [] unless job + + Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: root_variables, + to: transform_to_yaml_variables(job[:job_variables]), + inheritance: job.fetch(:root_variables_inheritance, true) + ) + end + + def stage_for(job_name) + jobs.dig(job_name, :stage) + end + private def variables diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b3dc59466ec..6159fb0a811 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -8,13 +8,13 @@ module Gitlab # LifecycleEvents lets Rails initializers register application startup hooks # that are sensitive to forking. For example, to defer the creation of # watchdog threads. This lets us abstract away the Unix process - # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # lifecycles of Sidekiq, Puma, Puma Cluster, etc. # # We have the following lifecycle events. # # - on_before_fork (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -22,7 +22,7 @@ module Gitlab # # - on_master_start (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -30,7 +30,7 @@ module Gitlab # # - on_before_blackout_period (on master process): # - # Unicorn/Puma Cluster: This will be called before a blackout + # Puma Cluster: This will be called before a blackout # period when performing graceful shutdown of master. # This is called on `master` process. # @@ -38,7 +38,7 @@ module Gitlab # # - on_before_graceful_shutdown (on master process): # - # Unicorn/Puma Cluster: This will be called before a graceful + # Puma Cluster: This will be called before a graceful # shutdown of workers starts happening, but after blackout period. # This is called on `master` process. # @@ -46,11 +46,6 @@ module Gitlab # # - on_before_master_restart (on master process): # - # Unicorn: This will be called before a new master is spun up. - # This is called on forked master before `execve` to become - # a new masterfor Unicorn. This means that this does not really - # affect old master process. - # # Puma Cluster: This will be called before a new master is spun up. # This is called on `master` process. # @@ -58,7 +53,7 @@ module Gitlab # # - on_worker_start (on worker process): # - # Unicorn/Puma Cluster: This is called in the worker process + # Puma Cluster: This is called in the worker process # exactly once before processing requests. # # Sidekiq/Puma Single: This is called immediately. @@ -114,7 +109,7 @@ module Gitlab end # - # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # Lifecycle integration methods (called from puma.rb, etc.) # def do_worker_start call(:worker_start_hooks, @worker_start_hooks) @@ -167,9 +162,6 @@ module Gitlab # Sidekiq doesn't fork return false if Gitlab::Runtime.sidekiq? - # Unicorn always forks - return true if Gitlab::Runtime.unicorn? - # Puma sometimes forks return true if in_clustered_puma? diff --git a/lib/gitlab/cluster/mixins/unicorn_http_server.rb b/lib/gitlab/cluster/mixins/unicorn_http_server.rb deleted file mode 100644 index 440ed02a355..00000000000 --- a/lib/gitlab/cluster/mixins/unicorn_http_server.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - module Mixins - module UnicornHttpServer - def self.prepended(base) - unless base.method_defined?(:reexec) && base.method_defined?(:stop) - raise 'missing method Unicorn::HttpServer#reexec or Unicorn::HttpServer#stop' - end - end - - def reexec - Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown - - super - end - - # The stop on non-graceful shutdown is executed twice: - # `#stop(false)` and `#stop`. - # - # The first stop will wipe-out all workers, so we need to check - # the flag and a list of workers - def stop(graceful = true) - if graceful && @workers.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables - Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown - end - - super - end - end - end - end -end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index fd9f58a34f3..e634291f894 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -12,12 +12,9 @@ module Gitlab require 'puma_worker_killer' PumaWorkerKiller.config do |config| - # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) - # Importantly RAM is for _all_workers (ie, the cluster), - # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX worker_count = puma_options[:workers] || 1 - # The Puma Worker Killer checks the total RAM used by both the master - # and worker processes. + # The Puma Worker Killer checks the total memory used by the cluster, + # i.e. both primary and worker processes. # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 # # Additional memory is added when running in `development` diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index e42b174e085..d7b31946ab0 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -24,7 +24,7 @@ module Gitlab 'media_src' => "'self'", 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com", 'style_src' => "'self' 'unsafe-inline'", - 'worker_src' => "'self'", + 'worker_src' => "'self' blob: data:", 'object_src' => "'none'", 'report_uri' => nil } @@ -79,6 +79,7 @@ module Gitlab append_to_directive(settings_hash, 'script_src', cdn_host) append_to_directive(settings_hash, 'style_src', cdn_host) + append_to_directive(settings_hash, 'font_src', cdn_host) end def self.append_to_directive(settings_hash, directive, text) diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index 7559cd376bf..b309802f296 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -3,10 +3,9 @@ module Gitlab module CycleAnalytics class StageSummary - def initialize(project, from:, to: nil, current_user:) + def initialize(project, options:, current_user:) @project = project - @from = from - @to = to + @options = options @current_user = current_user end @@ -20,15 +19,15 @@ module Gitlab private def issue_stats - serialize(Summary::Issue.new(project: @project, from: @from, to: @to, current_user: @current_user)) + serialize(Summary::Issue.new(project: @project, options: @options, current_user: @current_user)) end def commit_stats - serialize(Summary::Commit.new(project: @project, from: @from, to: @to)) + serialize(Summary::Commit.new(project: @project, options: @options)) end def deployments_summary - @deployments_summary ||= Summary::Deploy.new(project: @project, from: @from, to: @to) + @deployments_summary ||= Summary::Deploy.new(project: @project, options: @options) end def deploy_stats @@ -39,8 +38,7 @@ module Gitlab serialize( Summary::DeploymentFrequency.new( deployments: deployments_summary.value.raw_value, - from: @from, - to: @to), + options: @options), with_unit: true ) end @@ -50,8 +48,7 @@ module Gitlab end def serialize(summary_object, with_unit: false) - AnalyticsSummarySerializer.new.represent( - summary_object, with_unit: with_unit) + AnalyticsSummarySerializer.new.represent(summary_object, with_unit: with_unit) end end end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 67ad75652b0..50a8f189df0 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,10 +4,9 @@ module Gitlab module CycleAnalytics module Summary class Base - def initialize(project:, from:, to: nil) + def initialize(project:, options:) @project = project - @from = from - @to = to + @options = options end def title diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 1dc9d5de966..fb55c3df869 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -21,7 +21,7 @@ module Gitlab def commits_count return unless ref - @commits_count ||= gitaly_commit_client.commit_count(ref, after: @from, before: @to) + @commits_count ||= gitaly_commit_client.commit_count(ref, after: @options[:from], before: @options[:to]) end def gitaly_commit_client diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index e5bf6ef616f..ea16226a865 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -16,7 +16,7 @@ module Gitlab def deployments_count DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success, order_by: :finished_at) + .new(project: @project, finished_after: @options[:from], finished_before: @options[:to], status: :success, order_by: :finished_at) .execute .count end diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb index 00676a02a6f..1947866d772 100644 --- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -6,10 +6,10 @@ module Gitlab class DeploymentFrequency < Base include SummaryHelper - def initialize(deployments:, from:, to: nil, project: nil) + def initialize(deployments:, options:, project: nil) @deployments = deployments - super(project: project, from: from, to: to) + super(project: project, options: options) end def title @@ -17,7 +17,7 @@ module Gitlab end def value - @value ||= frequency(@deployments, @from, @to || Time.now) + @value ||= frequency(@deployments, @options[:from], @options[:to] || Time.current) end def unit diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 462fd4c2d3d..34e0d34b960 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -4,10 +4,9 @@ module Gitlab module CycleAnalytics module Summary class Issue < Base - def initialize(project:, from:, to: nil, current_user:) + def initialize(project:, options:, current_user:) @project = project - @from = from - @to = to + @options = options @current_user = current_user end @@ -23,10 +22,18 @@ module Gitlab def issues_count IssuesFinder - .new(@current_user, project_id: @project.id, created_after: @from, created_before: @to) + .new(@current_user, finder_params) .execute .count end + + def finder_params + @options.dup.tap do |hash| + hash[:created_after] = hash.delete(:from) + hash[:created_before] = hash.delete(:to) + hash[:project_id] = @project.id + end + end end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 4c31f986be5..91e6fc11a53 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -83,7 +83,9 @@ module Gitlab { id: runner.id, description: runner.description, + runner_type: runner.runner_type, active: runner.active?, + is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 766eaf54afe..4d70e3949dd 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -79,7 +79,9 @@ module Gitlab { id: runner.id, description: runner.description, + runner_type: runner.runner_type, active: runner.active?, + is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb index 8aee25e9fe6..87679654a17 100644 --- a/lib/gitlab/data_builder/wiki_page.rb +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -18,7 +18,8 @@ module Gitlab wiki: wiki.hook_attrs, object_attributes: wiki_page.hook_attrs.merge( url: Gitlab::UrlBuilder.build(wiki_page), - action: action + action: action, + diff_url: Gitlab::UrlBuilder.build(wiki_page, action: :diff, version_id: wiki_page.version.id) ) } end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 59249c8bc1f..aa419d75df2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -9,12 +9,12 @@ module Gitlab # 'old_name' => 'new_name' # }.freeze TABLES_TO_BE_RENAMED = { - 'analytics_instance_statistics_measurements' => 'analytics_usage_trends_measurements' + 'services' => 'integrations' }.freeze # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements - MINIMUM_POSTGRES_VERSION = 11 + MINIMUM_POSTGRES_VERSION = 12 # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html MAX_INT_VALUE = 2147483647 @@ -60,7 +60,7 @@ module Gitlab end def self.config - default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.config || {} + default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.configuration_hash || {} default_config_hash.with_indifferent_access.tap do |hash| # Match config/initializers/database_config.rb @@ -88,6 +88,11 @@ module Gitlab end end + # Disables prepared statements for the current database connection. + def self.disable_prepared_statements + ActiveRecord::Base.establish_connection(config.merge(prepared_statements: false)) + end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql') == 0 @@ -142,7 +147,7 @@ module Gitlab is required for this version of GitLab. <% if Rails.env.development? || Rails.env.test? %> If using gitlab-development-kit, please find the relevant steps here: - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/postgresql.md#upgrade-postgresql + https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql <% end %> Please upgrade your environment to a supported PostgreSQL version, see https://docs.gitlab.com/ee/install/requirements.html#database for details. @@ -288,7 +293,7 @@ module Gitlab # @param [ActiveRecord::Connection] ar_connection # @return [String] def self.get_write_location(ar_connection) - use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: false) + use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) sql = if use_new_load_balancer_query <<~NEWSQL diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 869b97b8ac0..9a1dc4ee17d 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -30,7 +30,7 @@ module Gitlab scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } - delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, + delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration attribute :pause_ms, :integer, default: 100 diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index e85162f355e..36e89023c86 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -14,12 +14,20 @@ module Gitlab class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', foreign_key: :batched_background_migration_id + validates :job_arguments, uniqueness: { + scope: [:job_class_name, :table_name, :column_name] + } + scope :queue_order, -> { order(id: :asc) } + scope :queued, -> { where(status: [:active, :paused]) } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do + where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) + .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals + end enum status: { paused: 0, active: 1, - aborted: 2, finished: 3, failed: 4 } @@ -30,6 +38,14 @@ module Gitlab active.queue_order.first end + def self.successful_rows_counts(migrations) + BatchedJob + .succeeded + .where(batched_background_migration_id: migrations) + .group(:batched_background_migration_id) + .sum(:batch_size) + end + def interval_elapsed?(variance: 0) return true unless last_job diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb index e99ea7a3232..17c16640e4c 100644 --- a/lib/gitlab/database/consistency.rb +++ b/lib/gitlab/database/consistency.rb @@ -4,28 +4,18 @@ module Gitlab module Database ## # This class is used to make it possible to ensure read consistency in - # GitLab EE without the need of overriding a lot of methods / classes / + # GitLab without the need of overriding a lot of methods / classes / # classs. # - # This is a CE class that does nothing in CE, because database load - # balancing is EE-only feature, but you can still use it in CE. It will - # start ensuring read consistency once it is overridden in EE. - # - # Using this class in CE helps to avoid creeping discrepancy between CE / - # EE only to force usage of the primary database in EE. - # class Consistency ## - # In CE there is no database load balancing, so all reads are expected to - # be consistent by the ACID guarantees of a single PostgreSQL instance. - # - # This method is overridden in EE. + # Within the block, disable the database load balancing for calls that + # require read consistency after recent writes. # def self.with_read_consistency(&block) - yield + ::Gitlab::Database::LoadBalancing::Session + .current.use_primary(&block) end end end end - -::Gitlab::Database::Consistency.singleton_class.prepend_mod_with('Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 892f8291780..7439591be99 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -11,6 +11,25 @@ module Gitlab self.inheritance_column = :_type_disabled end end + + def each_batch(table_name, scope: ->(table) { table.all }, of: 1000) + if transaction_open? + raise <<~MSG.squish + each_batch should not run inside a transaction, you can disable + transactions by calling disable_ddl_transaction! in the body of + your migration class + MSG + end + + scope.call(define_batchable_model(table_name)) + .each_batch(of: of) { |batch| yield batch } + end + + def each_batch_range(table_name, scope: ->(table) { table.all }, of: 1000) + each_batch(table_name, scope: scope, of: of) do |batch| + yield batch.pluck('MIN(id), MAX(id)').first + end + end end end end diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb new file mode 100644 index 00000000000..88743cd2e75 --- /dev/null +++ b/lib/gitlab/database/load_balancing.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # The exceptions raised for connection errors. + CONNECTION_ERRORS = if defined?(PG) + [ + PG::ConnectionBad, + PG::ConnectionDoesNotExist, + PG::ConnectionException, + PG::ConnectionFailure, + PG::UnableToSend, + # During a failover this error may be raised when + # writing to a primary. + PG::ReadOnlySqlTransaction + ].freeze + else + [].freeze + end + + ProxyNotConfiguredError = Class.new(StandardError) + + # The connection proxy to use for load balancing (if enabled). + def self.proxy + unless @proxy + Gitlab::ErrorTracking.track_exception( + ProxyNotConfiguredError.new( + "Attempting to access the database load balancing proxy, but it wasn't configured.\n" \ + "Did you forget to call '#{self.name}.configure_proxy'?" + )) + end + + @proxy + end + + # Returns a Hash containing the load balancing configuration. + def self.configuration + Gitlab::Database.config[:load_balancing] || {} + end + + # Returns the maximum replica lag size in bytes. + def self.max_replication_difference + (configuration['max_replication_difference'] || 8.megabytes).to_i + end + + # Returns the maximum lag time for a replica. + def self.max_replication_lag_time + (configuration['max_replication_lag_time'] || 60.0).to_f + end + + # Returns the interval (in seconds) to use for checking the status of a + # replica. + def self.replica_check_interval + (configuration['replica_check_interval'] || 60).to_f + end + + # Returns the additional hosts to use for load balancing. + def self.hosts + configuration['hosts'] || [] + end + + def self.service_discovery_enabled? + configuration.dig('discover', 'record').present? + end + + def self.service_discovery_configuration + conf = configuration['discover'] || {} + + { + nameserver: conf['nameserver'] || 'localhost', + port: conf['port'] || 8600, + record: conf['record'], + record_type: conf['record_type'] || 'A', + interval: conf['interval'] || 60, + disconnect_timeout: conf['disconnect_timeout'] || 120, + use_tcp: conf['use_tcp'] || false + } + end + + def self.pool_size + Gitlab::Database.config[:pool] + end + + # Returns true if load balancing is to be enabled. + def self.enable? + return false if Gitlab::Runtime.rake? + return false if Gitlab::Runtime.sidekiq? && !Gitlab::Utils.to_boolean(ENV['ENABLE_LOAD_BALANCING_FOR_SIDEKIQ'], default: false) + return false unless self.configured? + + true + end + + # Returns true if load balancing has been configured. Since + # Sidekiq does not currently use load balancing, we + # may want Web application servers to detect replication lag by + # posting the write location of the database if load balancing is + # configured. + def self.configured? + hosts.any? || service_discovery_enabled? + end + + def self.start_service_discovery + return unless service_discovery_enabled? + + ServiceDiscovery.new(service_discovery_configuration).start + end + + # Configures proxying of requests. + def self.configure_proxy(proxy = ConnectionProxy.new(hosts)) + @proxy = proxy + + # This hijacks the "connection" method to ensure both + # `ActiveRecord::Base.connection` and all models use the same load + # balancing proxy. + ActiveRecord::Base.singleton_class.prepend(ActiveRecordProxy) + end + + def self.active_record_models + ActiveRecord::Base.descendants + end + + DB_ROLES = [ + ROLE_PRIMARY = :primary, + ROLE_REPLICA = :replica, + ROLE_UNKNOWN = :unknown + ].freeze + + # Returns the role (primary/replica) of the database the connection is + # connecting to. At the moment, the connection can only be retrieved by + # Gitlab::Database::LoadBalancer#read or #read_write or from the + # ActiveRecord directly. Therefore, if the load balancer doesn't + # recognize the connection, this method returns the primary role + # directly. In future, we may need to check for other sources. + def self.db_role_for_connection(connection) + return ROLE_PRIMARY if !enable? || @proxy.blank? + + proxy.load_balancer.db_role_for_connection(connection) + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/active_record_proxy.rb b/lib/gitlab/database/load_balancing/active_record_proxy.rb new file mode 100644 index 00000000000..7763497e770 --- /dev/null +++ b/lib/gitlab/database/load_balancing/active_record_proxy.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Module injected into ActiveRecord::Base to allow hijacking of the + # "connection" method. + module ActiveRecordProxy + def connection + LoadBalancing.proxy + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb new file mode 100644 index 00000000000..3a09689a724 --- /dev/null +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +# rubocop:disable GitlabSecurity/PublicSend + +module Gitlab + module Database + module LoadBalancing + # Redirecting of ActiveRecord connections. + # + # The ConnectionProxy class redirects ActiveRecord connection requests to + # the right load balancer pool, depending on the type of query. + class ConnectionProxy + WriteInsideReadOnlyTransactionError = Class.new(StandardError) + READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction + + attr_reader :load_balancer + + # These methods perform writes after which we need to stick to the + # primary. + STICKY_WRITES = %i( + delete + delete_all + insert + update + update_all + ).freeze + + NON_STICKY_READS = %i( + sanitize_limit + select + select_one + select_rows + quote_column_name + ).freeze + + # hosts - The hosts to use for load balancing. + def initialize(hosts = []) + @load_balancer = LoadBalancer.new(hosts) + end + + def select_all(arel, name = nil, binds = [], preparable: nil) + if arel.respond_to?(:locked) && arel.locked + # SELECT ... FOR UPDATE queries should be sent to the primary. + write_using_load_balancer(:select_all, [arel, name, binds], + sticky: true) + else + read_using_load_balancer(:select_all, [arel, name, binds]) + end + end + + NON_STICKY_READS.each do |name| + define_method(name) do |*args, &block| + read_using_load_balancer(name, args, &block) + end + end + + STICKY_WRITES.each do |name| + define_method(name) do |*args, &block| + write_using_load_balancer(name, args, sticky: true, &block) + end + end + + def transaction(*args, &block) + if current_session.fallback_to_replicas_for_ambiguous_queries? + track_read_only_transaction! + read_using_load_balancer(:transaction, args, &block) + else + write_using_load_balancer(:transaction, args, sticky: true, &block) + end + + ensure + untrack_read_only_transaction! + end + + # Delegates all unknown messages to a read-write connection. + def method_missing(name, *args, &block) + if current_session.fallback_to_replicas_for_ambiguous_queries? + read_using_load_balancer(name, args, &block) + else + write_using_load_balancer(name, args, &block) + end + end + + # Performs a read using the load balancer. + # + # name - The name of the method to call on a connection object. + def read_using_load_balancer(name, args, &block) + if current_session.use_primary? && + !current_session.use_replicas_for_read_queries? + @load_balancer.read_write do |connection| + connection.send(name, *args, &block) + end + else + @load_balancer.read do |connection| + connection.send(name, *args, &block) + end + end + end + + # Performs a write using the load balancer. + # + # name - The name of the method to call on a connection object. + # sticky - If set to true the session will stick to the master after + # the write. + def write_using_load_balancer(name, args, sticky: false, &block) + if read_only_transaction? + raise WriteInsideReadOnlyTransactionError, 'A write query is performed inside a read-only transaction' + end + + @load_balancer.read_write do |connection| + # Sticking has to be enabled before calling the method. Not doing so + # could lead to methods called in a block still being performed on a + # secondary instead of on a primary (when necessary). + current_session.write! if sticky + + connection.send(name, *args, &block) + end + end + + private + + def current_session + ::Gitlab::Database::LoadBalancing::Session.current + end + + def track_read_only_transaction! + Thread.current[READ_ONLY_TRANSACTION_KEY] = true + end + + def untrack_read_only_transaction! + Thread.current[READ_ONLY_TRANSACTION_KEY] = nil + end + + def read_only_transaction? + Thread.current[READ_ONLY_TRANSACTION_KEY] == true + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb new file mode 100644 index 00000000000..3e74b5ea727 --- /dev/null +++ b/lib/gitlab/database/load_balancing/host.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # A single database host used for load balancing. + class Host + attr_reader :pool, :last_checked_at, :intervals, :load_balancer, :host, :port + + delegate :connection, :release_connection, :enable_query_cache!, :disable_query_cache!, :query_cache_enabled, to: :pool + + CONNECTION_ERRORS = + if defined?(PG) + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid, + PG::Error + ].freeze + else + [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid + ].freeze + end + + # host - The address of the database. + # load_balancer - The LoadBalancer that manages this Host. + def initialize(host, load_balancer, port: nil) + @host = host + @port = port + @load_balancer = load_balancer + @pool = Database.create_connection_pool(LoadBalancing.pool_size, host, port) + @online = true + @last_checked_at = Time.zone.now + + interval = LoadBalancing.replica_check_interval + @intervals = (interval..(interval * 2)).step(0.5).to_a + end + + # Disconnects the pool, once all connections are no longer in use. + # + # timeout - The time after which the pool should be forcefully + # disconnected. + def disconnect!(timeout = 120) + start_time = Metrics::System.monotonic_time + + while (Metrics::System.monotonic_time - start_time) <= timeout + break if pool.connections.none?(&:in_use?) + + sleep(2) + end + + pool.disconnect! + end + + def offline! + LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Marking host as offline', + db_host: @host, + db_port: @port + ) + + @online = false + @pool.disconnect! + end + + # Returns true if the host is online. + def online? + return @online unless check_replica_status? + + refresh_status + + if @online + LoadBalancing::Logger.info( + event: :host_online, + message: 'Host is online after replica status check', + db_host: @host, + db_port: @port + ) + else + LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Host is offline after replica status check', + db_host: @host, + db_port: @port + ) + end + + @online + rescue *CONNECTION_ERRORS + offline! + false + end + + def refresh_status + @online = replica_is_up_to_date? + @last_checked_at = Time.zone.now + end + + def check_replica_status? + (Time.zone.now - last_checked_at) >= intervals.sample + end + + def replica_is_up_to_date? + replication_lag_below_threshold? || data_is_recent_enough? + end + + def replication_lag_below_threshold? + if (lag_time = replication_lag_time) + lag_time <= LoadBalancing.max_replication_lag_time + else + false + end + end + + # Returns true if the replica has replicated enough data to be useful. + def data_is_recent_enough? + # It's possible for a replica to not replay WAL data for a while, + # despite being up to date. This can happen when a primary does not + # receive any writes for a while. + # + # To prevent this from happening we check if the lag size (in bytes) + # of the replica is small enough for the replica to be useful. We + # only do this if we haven't replicated in a while so we only need + # to connect to the primary when truly necessary. + if (lag_size = replication_lag_size) + lag_size <= LoadBalancing.max_replication_difference + else + false + end + end + + # Returns the replication lag time of this secondary in seconds as a + # float. + # + # This method will return nil if no lag time could be calculated. + def replication_lag_time + row = query_and_release('SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::float as lag') + + row['lag'].to_f if row.any? + end + + # Returns the number of bytes this secondary is lagging behind the + # primary. + # + # This method will return nil if no lag size could be calculated. + def replication_lag_size + location = connection.quote(primary_write_location) + row = query_and_release(<<-SQL.squish) + SELECT pg_wal_lsn_diff(#{location}, pg_last_wal_replay_lsn())::float + AS diff + SQL + + row['diff'].to_i if row.any? + rescue *CONNECTION_ERRORS + nil + end + + def primary_write_location + load_balancer.primary_write_location + ensure + load_balancer.release_primary_connection + end + + def database_replica_location + row = query_and_release(<<-SQL.squish) + SELECT pg_last_wal_replay_lsn()::text AS location + SQL + + row['location'] if row.any? + rescue *CONNECTION_ERRORS + nil + end + + # Returns true if this host has caught up to the given transaction + # write location. + # + # location - The transaction write location as reported by a primary. + def caught_up?(location) + string = connection.quote(location) + + # In case the host is a primary pg_last_wal_replay_lsn/pg_last_xlog_replay_location() returns + # NULL. The recovery check ensures we treat the host as up-to-date in + # such a case. + query = <<-SQL.squish + SELECT NOT pg_is_in_recovery() + OR pg_wal_lsn_diff(pg_last_wal_replay_lsn(), #{string}) >= 0 + AS result + SQL + + row = query_and_release(query) + + ::Gitlab::Utils.to_boolean(row['result']) + rescue *CONNECTION_ERRORS + false + end + + def query_and_release(sql) + connection.select_all(sql).first || {} + rescue StandardError + {} + ensure + release_connection + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/lib/gitlab/database/load_balancing/host_list.rb new file mode 100644 index 00000000000..24800012947 --- /dev/null +++ b/lib/gitlab/database/load_balancing/host_list.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # A list of database hosts to use for connections. + class HostList + # hosts - The list of secondary hosts to add. + def initialize(hosts = []) + @hosts = hosts.shuffle + @pools = Set.new + @index = 0 + @mutex = Mutex.new + @hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts') + + set_metrics! + update_pools + end + + def hosts + @mutex.synchronize { @hosts.dup } + end + + def shuffle + @mutex.synchronize do + unsafe_shuffle + end + end + + def length + @mutex.synchronize { @hosts.length } + end + + def host_names_and_ports + @mutex.synchronize { @hosts.map { |host| [host.host, host.port] } } + end + + def manage_pool?(pool) + @pools.include?(pool) + end + + def hosts=(hosts) + @mutex.synchronize do + @hosts = hosts + unsafe_shuffle + update_pools + end + + set_metrics! + end + + # Sets metrics before returning next host + def next + next_host.tap do |_| + set_metrics! + end + end + + private + + def unsafe_shuffle + @hosts = @hosts.shuffle + @index = 0 + end + + # Returns the next available host. + # + # Returns a Gitlab::Database::LoadBalancing::Host instance, or nil if no + # hosts were available. + def next_host + @mutex.synchronize do + break if @hosts.empty? + + started_at = @index + + loop do + host = @hosts[@index] + @index = (@index + 1) % @hosts.length + + break host if host.online? + + # Return nil once we have cycled through all hosts and none were + # available. + break if @index == started_at + end + end + end + + def set_metrics! + @hosts_gauge.set({}, @hosts.length) + end + + def update_pools + @pools = Set.new(@hosts.map(&:pool)) + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb new file mode 100644 index 00000000000..a833bb8491f --- /dev/null +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -0,0 +1,275 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Load balancing for ActiveRecord connections. + # + # Each host in the load balancer uses the same credentials as the primary + # database. + # + # This class *requires* that `ActiveRecord::Base.retrieve_connection` + # always returns a connection to the primary. + class LoadBalancer + CACHE_KEY = :gitlab_load_balancer_host + VALID_HOSTS_CACHE_KEY = :gitlab_load_balancer_valid_hosts + + attr_reader :host_list + + # hosts - The hostnames/addresses of the additional databases. + def initialize(hosts = []) + @host_list = HostList.new(hosts.map { |addr| Host.new(addr, self) }) + @connection_db_roles = {}.compare_by_identity + @connection_db_roles_count = {}.compare_by_identity + end + + # Yields a connection that can be used for reads. + # + # If no secondaries were available this method will use the primary + # instead. + def read(&block) + connection = nil + conflict_retried = 0 + + while host + ensure_caching! + + begin + connection = host.connection + track_connection_role(connection, ROLE_REPLICA) + + return yield connection + rescue StandardError => error + untrack_connection_role(connection) + + if serialization_failure?(error) + # This error can occur when a query conflicts. See + # https://www.postgresql.org/docs/current/static/hot-standby.html#HOT-STANDBY-CONFLICT + # for more information. + # + # In this event we'll cycle through the secondaries at most 3 + # times before using the primary instead. + will_retry = conflict_retried < @host_list.length * 3 + + LoadBalancing::Logger.warn( + event: :host_query_conflict, + message: 'Query conflict on host', + conflict_retried: conflict_retried, + will_retry: will_retry, + db_host: host.host, + db_port: host.port, + host_list_length: @host_list.length + ) + + if will_retry + conflict_retried += 1 + release_host + else + break + end + elsif connection_error?(error) + host.offline! + release_host + else + raise error + end + end + end + + LoadBalancing::Logger.warn( + event: :no_secondaries_available, + message: 'No secondaries were available, using primary instead', + conflict_retried: conflict_retried, + host_list_length: @host_list.length + ) + + read_write(&block) + ensure + untrack_connection_role(connection) + end + + # Yields a connection that can be used for both reads and writes. + def read_write + connection = nil + # In the event of a failover the primary may be briefly unavailable. + # Instead of immediately grinding to a halt we'll retry the operation + # a few times. + retry_with_backoff do + connection = ActiveRecord::Base.retrieve_connection + track_connection_role(connection, ROLE_PRIMARY) + + yield connection + end + ensure + untrack_connection_role(connection) + end + + # Recognize the role (primary/replica) of the database this connection + # is connecting to. If the connection is not issued by this load + # balancer, return nil + def db_role_for_connection(connection) + return @connection_db_roles[connection] if @connection_db_roles[connection] + return ROLE_REPLICA if @host_list.manage_pool?(connection.pool) + return ROLE_PRIMARY if connection.pool == ActiveRecord::Base.connection_pool + end + + # Returns a host to use for queries. + # + # Hosts are scoped per thread so that multiple threads don't + # accidentally re-use the same host + connection. + def host + RequestStore[CACHE_KEY] ||= current_host_list.next + end + + # Releases the host and connection for the current thread. + def release_host + if host = RequestStore[CACHE_KEY] + host.disable_query_cache! + host.release_connection + end + + RequestStore.delete(CACHE_KEY) + RequestStore.delete(VALID_HOSTS_CACHE_KEY) + end + + def release_primary_connection + ActiveRecord::Base.connection_pool.release_connection + end + + # Returns the transaction write location of the primary. + def primary_write_location + location = read_write do |connection| + ::Gitlab::Database.get_write_location(connection) + end + + return location if location + + raise 'Failed to determine the write location of the primary database' + end + + # Returns true if all hosts have caught up to the given transaction + # write location. + def all_caught_up?(location) + @host_list.hosts.all? { |host| host.caught_up?(location) } + end + + # Returns true if there was at least one host that has caught up with the given transaction. + # + # In case of a retry, this method also stores the set of hosts that have caught up. + def select_caught_up_hosts(location) + all_hosts = @host_list.hosts + valid_hosts = all_hosts.select { |host| host.caught_up?(location) } + + return false if valid_hosts.empty? + + # Hosts can come online after the time when this scan was done, + # so we need to remember the ones that can be used. If the host went + # offline, we'll just rely on the retry mechanism to use the primary. + set_consistent_hosts_for_request(HostList.new(valid_hosts)) + + # Since we will be using a subset from the original list, let's just + # pick a random host and mix up the original list to ensure we don't + # only end up using one replica. + RequestStore[CACHE_KEY] = valid_hosts.sample + @host_list.shuffle + + true + end + + # Returns true if there was at least one host that has caught up with the given transaction. + # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use. + # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any. + def select_up_to_date_host(location) + all_hosts = @host_list.hosts.shuffle + host = all_hosts.find { |host| host.caught_up?(location) } + + return false unless host + + RequestStore[CACHE_KEY] = host + + true + end + + def set_consistent_hosts_for_request(hosts) + RequestStore[VALID_HOSTS_CACHE_KEY] = hosts + end + + # Yields a block, retrying it upon error using an exponential backoff. + def retry_with_backoff(retries = 3, time = 2) + retried = 0 + last_error = nil + + while retried < retries + begin + return yield + rescue StandardError => error + raise error unless connection_error?(error) + + # We need to release the primary connection as otherwise Rails + # will keep raising errors when using the connection. + release_primary_connection + + last_error = error + sleep(time) + retried += 1 + time **= 2 + end + end + + raise last_error + end + + def connection_error?(error) + case error + when ActiveRecord::StatementInvalid, ActionView::Template::Error + # After connecting to the DB Rails will wrap query errors using this + # class. + connection_error?(error.cause) + when *CONNECTION_ERRORS + true + else + # When PG tries to set the client encoding but fails due to a + # connection error it will raise a PG::Error instance. Catching that + # would catch all errors (even those we don't want), so instead we + # check for the message of the error. + error.message.start_with?('invalid encoding name:') + end + end + + def serialization_failure?(error) + if error.cause + serialization_failure?(error.cause) + else + error.is_a?(PG::TRSerializationFailure) + end + end + + private + + def ensure_caching! + host.enable_query_cache! unless host.query_cache_enabled + end + + def track_connection_role(connection, role) + @connection_db_roles[connection] = role + @connection_db_roles_count[connection] ||= 0 + @connection_db_roles_count[connection] += 1 + end + + def untrack_connection_role(connection) + return if connection.blank? || @connection_db_roles_count[connection].blank? + + @connection_db_roles_count[connection] -= 1 + if @connection_db_roles_count[connection] <= 0 + @connection_db_roles.delete(connection) + @connection_db_roles_count.delete(connection) + end + end + + def current_host_list + RequestStore[VALID_HOSTS_CACHE_KEY] || @host_list + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/logger.rb b/lib/gitlab/database/load_balancing/logger.rb new file mode 100644 index 00000000000..ee67ffcc99c --- /dev/null +++ b/lib/gitlab/database/load_balancing/logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'database_load_balancing' + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb new file mode 100644 index 00000000000..4734ff99bd3 --- /dev/null +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Rack middleware to handle sticking when serving Rails requests. Grape + # API calls are handled separately as different API endpoints need to + # stick based on different objects. + class RackMiddleware + STICK_OBJECT = 'load_balancing.stick_object' + + # Unsticks or continues sticking the current request. + # + # This method also updates the Rack environment so #call can later + # determine if we still need to stick or not. + # + # env - The Rack environment. + # namespace - The namespace to use for sticking. + # id - The identifier to use for sticking. + def self.stick_or_unstick(env, namespace, id) + return unless LoadBalancing.enable? + + Sticking.unstick_or_continue_sticking(namespace, id) + + env[STICK_OBJECT] ||= Set.new + env[STICK_OBJECT] << [namespace, id] + end + + def initialize(app) + @app = app + end + + def call(env) + # Ensure that any state that may have run before the first request + # doesn't linger around. + clear + + unstick_or_continue_sticking(env) + + result = @app.call(env) + + stick_if_necessary(env) + + result + ensure + clear + end + + # Determine if we need to stick based on currently available user data. + # + # Typically this code will only be reachable for Rails requests as + # Grape data is not yet available at this point. + def unstick_or_continue_sticking(env) + namespaces_and_ids = sticking_namespaces_and_ids(env) + + namespaces_and_ids.each do |namespace, id| + Sticking.unstick_or_continue_sticking(namespace, id) + end + end + + # Determine if we need to stick after handling a request. + def stick_if_necessary(env) + namespaces_and_ids = sticking_namespaces_and_ids(env) + + namespaces_and_ids.each do |namespace, id| + Sticking.stick_if_necessary(namespace, id) + end + end + + def clear + load_balancer.release_host + Session.clear_session + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + # Determines the sticking namespace and identifier based on the Rack + # environment. + # + # For Rails requests this uses warden, but Grape and others have to + # manually set the right environment variable. + def sticking_namespaces_and_ids(env) + warden = env['warden'] + + if warden && warden.user + [[:user, warden.user.id]] + elsif env[STICK_OBJECT].present? + env[STICK_OBJECT].to_a + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/resolver.rb b/lib/gitlab/database/load_balancing/resolver.rb new file mode 100644 index 00000000000..a291080cc3d --- /dev/null +++ b/lib/gitlab/database/load_balancing/resolver.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'net/dns' +require 'resolv' + +module Gitlab + module Database + module LoadBalancing + class Resolver + UnresolvableNameserverError = Class.new(StandardError) + + def initialize(nameserver) + @nameserver = nameserver + end + + def resolve + address = ip_address || ip_address_from_hosts_file || + ip_address_from_dns + + unless address + raise UnresolvableNameserverError, + "could not resolve #{@nameserver}" + end + + address + end + + private + + def ip_address + IPAddr.new(@nameserver) + rescue IPAddr::InvalidAddressError + end + + def ip_address_from_hosts_file + ip = Resolv::Hosts.new.getaddress(@nameserver) + IPAddr.new(ip) + rescue Resolv::ResolvError + end + + def ip_address_from_dns + answer = Net::DNS::Resolver.start(@nameserver, Net::DNS::A).answer + return if answer.empty? + + answer.first.address + rescue Net::DNS::Resolver::NoResponseError + raise UnresolvableNameserverError, "no response from DNS server(s)" + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb new file mode 100644 index 00000000000..9b42b25be1c --- /dev/null +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +require 'net/dns' +require 'resolv' + +module Gitlab + module Database + module LoadBalancing + # Service discovery of secondary database hosts. + # + # Service discovery works by periodically looking up a DNS record. If the + # DNS record returns a new list of hosts, this class will update the load + # balancer with said hosts. Requests may continue to use the old hosts + # until they complete. + class ServiceDiscovery + attr_reader :interval, :record, :record_type, :disconnect_timeout + + MAX_SLEEP_ADJUSTMENT = 10 + + RECORD_TYPES = { + 'A' => Net::DNS::A, + 'SRV' => Net::DNS::SRV + }.freeze + + Address = Struct.new(:hostname, :port) do + def to_s + port ? "#{hostname}:#{port}" : hostname + end + + def <=>(other) + self.to_s <=> other.to_s + end + end + + # nameserver - The nameserver to use for DNS lookups. + # port - The port of the nameserver. + # record - The DNS record to look up for retrieving the secondaries. + # record_type - The type of DNS record to look up + # interval - The time to wait between lookups. + # disconnect_timeout - The time after which an old host should be + # forcefully disconnected. + # use_tcp - Use TCP instaed of UDP to look up resources + def initialize(nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, use_tcp: false) + @nameserver = nameserver + @port = port + @record = record + @record_type = record_type_for(record_type) + @interval = interval + @disconnect_timeout = disconnect_timeout + @use_tcp = use_tcp + end + + def start + Thread.new do + loop do + interval = + begin + refresh_if_necessary + rescue StandardError => error + # Any exceptions that might occur should be reported to + # Sentry, instead of silently terminating this thread. + Gitlab::ErrorTracking.track_exception(error) + + Gitlab::AppLogger.error( + "Service discovery encountered an error: #{error.message}" + ) + + self.interval + end + + # We slightly randomize the sleep() interval. This should reduce + # the likelihood of _all_ processes refreshing at the same time, + # possibly putting unnecessary pressure on the DNS server. + sleep(interval + rand(MAX_SLEEP_ADJUSTMENT)) + end + end + end + + # Refreshes the hosts, but only if the DNS record returned a new list of + # addresses. + # + # The return value is the amount of time (in seconds) to wait before + # checking the DNS record for any changes. + def refresh_if_necessary + interval, from_dns = addresses_from_dns + + current = addresses_from_load_balancer + + replace_hosts(from_dns) if from_dns != current + + interval + end + + # Replaces all the hosts in the load balancer with the new ones, + # disconnecting the old connections. + # + # addresses - An Array of Address structs to use for the new hosts. + def replace_hosts(addresses) + old_hosts = load_balancer.host_list.hosts + + load_balancer.host_list.hosts = addresses.map do |addr| + Host.new(addr.hostname, load_balancer, port: addr.port) + end + + # We must explicitly disconnect the old connections, otherwise we may + # leak database connections over time. For example, if a request + # started just before we added the new hosts it will use an old + # host/connection. While this connection will be checked in and out, + # it won't be explicitly disconnected. + old_hosts.each do |host| + host.disconnect!(disconnect_timeout) + end + end + + # Returns an Array containing: + # + # 1. The time to wait for the next check. + # 2. An array containing the hostnames of the DNS record. + def addresses_from_dns + response = resolver.search(record, record_type) + resources = response.answer + + addresses = + case record_type + when Net::DNS::A + addresses_from_a_record(resources) + when Net::DNS::SRV + addresses_from_srv_record(response) + end + + # Addresses are sorted so we can directly compare the old and new + # addresses, without having to use any additional data structures. + [new_wait_time_for(resources), addresses.sort] + end + + def new_wait_time_for(resources) + wait = resources.first&.ttl || interval + + # The preconfigured interval acts as a minimum amount of time to + # wait. + wait < interval ? interval : wait + end + + def addresses_from_load_balancer + load_balancer.host_list.host_names_and_ports.map do |hostname, port| + Address.new(hostname, port) + end.sort + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + def resolver + @resolver ||= Net::DNS::Resolver.new( + nameservers: Resolver.new(@nameserver).resolve, + port: @port, + use_tcp: @use_tcp + ) + end + + private + + def record_type_for(type) + RECORD_TYPES.fetch(type) do + raise(ArgumentError, "Unsupported record type: #{type}") + end + end + + def addresses_from_srv_record(response) + srv_resolver = SrvResolver.new(resolver, response.additional) + + response.answer.map do |r| + address = srv_resolver.address_for(r.host.to_s) + next unless address + + Address.new(address.to_s, r.port) + end.compact + end + + def addresses_from_a_record(resources) + resources.map { |r| Address.new(r.address.to_s) } + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/session.rb b/lib/gitlab/database/load_balancing/session.rb new file mode 100644 index 00000000000..3682c9265c2 --- /dev/null +++ b/lib/gitlab/database/load_balancing/session.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Tracking of load balancing state per user session. + # + # A session starts at the beginning of a request and ends once the request + # has been completed. Sessions can be used to keep track of what hosts + # should be used for queries. + class Session + CACHE_KEY = :gitlab_load_balancer_session + + def self.current + RequestStore[CACHE_KEY] ||= new + end + + def self.clear_session + RequestStore.delete(CACHE_KEY) + end + + def self.without_sticky_writes(&block) + current.ignore_writes(&block) + end + + def initialize + @use_primary = false + @performed_write = false + @ignore_writes = false + @fallback_to_replicas_for_ambiguous_queries = false + @use_replicas_for_read_queries = false + end + + def use_primary? + @use_primary + end + + alias_method :using_primary?, :use_primary? + + def use_primary! + @use_primary = true + end + + def use_primary(&blk) + used_primary = @use_primary + @use_primary = true + yield + ensure + @use_primary = used_primary || @performed_write + end + + def ignore_writes(&block) + @ignore_writes = true + + yield + ensure + @ignore_writes = false + end + + # Indicates that the read SQL statements from anywhere inside this + # blocks should use a replica, regardless of the current primary + # stickiness or whether a write query is already performed in the + # current session. This interface is reserved mostly for performance + # purpose. This is a good tool to push expensive queries, which can + # tolerate the replica lags, to the replicas. + # + # Write and ambiguous queries inside this block are still handled by + # the primary. + def use_replicas_for_read_queries(&blk) + previous_flag = @use_replicas_for_read_queries + @use_replicas_for_read_queries = true + yield + ensure + @use_replicas_for_read_queries = previous_flag + end + + def use_replicas_for_read_queries? + @use_replicas_for_read_queries == true + end + + # Indicate that the ambiguous SQL statements from anywhere inside this + # block should use a replica. The ambiguous statements include: + # - Transactions. + # - Custom queries (via exec_query, execute, etc.) + # - In-flight connection configuration change (SET LOCAL statement_timeout = 5000) + # + # This is a weak enforcement. This helper incorporates well with + # primary stickiness: + # - If the queries are about to write + # - The current session already performed writes + # - It prefers to use primary, aka, use_primary or use_primary! were called + def fallback_to_replicas_for_ambiguous_queries(&blk) + previous_flag = @fallback_to_replicas_for_ambiguous_queries + @fallback_to_replicas_for_ambiguous_queries = true + yield + ensure + @fallback_to_replicas_for_ambiguous_queries = previous_flag + end + + def fallback_to_replicas_for_ambiguous_queries? + @fallback_to_replicas_for_ambiguous_queries == true && !use_primary? && !performed_write? + end + + def write! + @performed_write = true + + return if @ignore_writes + + use_primary! + end + + def performed_write? + @performed_write + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb new file mode 100644 index 00000000000..524d69c00c0 --- /dev/null +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class SidekiqClientMiddleware + def call(worker_class, job, _queue, _redis_pool) + worker_class = worker_class.to_s.safe_constantize + + mark_data_consistency_location(worker_class, job) + + yield + end + + private + + def mark_data_consistency_location(worker_class, job) + # Mailers can't be constantized + return unless worker_class + return unless worker_class.include?(::ApplicationWorker) + return unless worker_class.get_data_consistency_feature_flag_enabled? + + return if location_already_provided?(job) + + job['worker_data_consistency'] = worker_class.get_data_consistency + + return unless worker_class.utilizes_load_balancing_capabilities? + + if Session.current.use_primary? + job['database_write_location'] = load_balancer.primary_write_location + else + job['database_replica_location'] = load_balancer.host.database_replica_location + end + end + + def location_already_provided?(job) + job['database_replica_location'] || job['database_write_location'] + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb new file mode 100644 index 00000000000..9bd0adf8dbd --- /dev/null +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class SidekiqServerMiddleware + JobReplicaNotUpToDate = Class.new(StandardError) + + def call(worker, job, _queue) + if requires_primary?(worker.class, job) + Session.current.use_primary! + end + + yield + ensure + clear + end + + private + + def clear + load_balancer.release_host + Session.clear_session + end + + def requires_primary?(worker_class, job) + return true unless worker_class.include?(::ApplicationWorker) + return true unless worker_class.utilizes_load_balancing_capabilities? + return true unless worker_class.get_data_consistency_feature_flag_enabled? + + location = job['database_write_location'] || job['database_replica_location'] + + return true unless location + + job_data_consistency = worker_class.get_data_consistency + job[:data_consistency] = job_data_consistency.to_s + + if replica_caught_up?(location) + job[:database_chosen] = 'replica' + false + elsif job_data_consistency == :delayed && not_yet_retried?(job) + job[:database_chosen] = 'retry' + raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ + " Replica was not up to date." + else + job[:database_chosen] = 'primary' + true + end + end + + def not_yet_retried?(job) + # if `retry_count` is `nil` it indicates that this job was never retried + # the `0` indicates that this is a first retry + job['retry_count'].nil? + end + + def load_balancer + LoadBalancing.proxy.load_balancer + end + + def replica_caught_up?(location) + if Feature.enabled?(:sidekiq_load_balancing_rotate_up_to_date_replica) + load_balancer.select_up_to_date_host(location) + else + load_balancer.host.caught_up?(location) + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/srv_resolver.rb b/lib/gitlab/database/load_balancing/srv_resolver.rb new file mode 100644 index 00000000000..20da525f4d2 --- /dev/null +++ b/lib/gitlab/database/load_balancing/srv_resolver.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Hostnames returned in SRV records cannot sometimes be resolved by a local + # resolver, however, there's a possibility that their A/AAAA records are + # returned as part of the SRV query in the additional section, so we try + # to extract the IPs from there first, failing back to querying the + # hostnames A/AAAA records one by one, using the same resolver that + # queried the SRV record. + class SrvResolver + include Gitlab::Utils::StrongMemoize + + attr_reader :resolver, :additional + + def initialize(resolver, additional) + @resolver = resolver + @additional = additional + end + + def address_for(host) + addresses_from_additional[host] || resolve_host(host) + end + + private + + def addresses_from_additional + strong_memoize(:addresses_from_additional) do + additional.each_with_object({}) do |rr, h| + h[rr.name] = rr.address if rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) + end + end + end + + def resolve_host(host) + record = resolver.search(host, Net::DNS::ANY).answer.find do |rr| + rr.is_a?(Net::DNS::RR::A) || rr.is_a?(Net::DNS::RR::AAAA) + end + + record&.address + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb new file mode 100644 index 00000000000..efbd7099300 --- /dev/null +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Module used for handling sticking connections to a primary, if + # necessary. + # + # ## Examples + # + # Sticking a user to the primary: + # + # Sticking.stick_if_necessary(:user, current_user.id) + # + # To unstick if possible, or continue using the primary otherwise: + # + # Sticking.unstick_or_continue_sticking(:user, current_user.id) + module Sticking + # The number of seconds after which a session should stop reading from + # the primary. + EXPIRATION = 30 + + # Sticks to the primary if a write was performed. + def self.stick_if_necessary(namespace, id) + return unless LoadBalancing.enable? + + stick(namespace, id) if Session.current.performed_write? + end + + # Checks if we are caught-up with all the work + def self.all_caught_up?(namespace, id) + location = last_write_location_for(namespace, id) + + return true unless location + + load_balancer.all_caught_up?(location).tap do |caught_up| + unstick(namespace, id) if caught_up + end + end + + # Selects hosts that have caught up with the primary. This ensures + # atomic selection of the host to prevent the host list changing + # in another thread. + # + # Returns true if one host was selected. + def self.select_caught_up_replicas(namespace, id) + location = last_write_location_for(namespace, id) + + # Unlike all_caught_up?, we return false if no write location exists. + # We want to be sure we talk to a replica that has caught up for a specific + # write location. If no such location exists, err on the side of caution. + return false unless location + + load_balancer.select_caught_up_hosts(location).tap do |selected| + unstick(namespace, id) if selected + end + end + + # Sticks to the primary if necessary, otherwise unsticks an object (if + # it was previously stuck to the primary). + def self.unstick_or_continue_sticking(namespace, id) + Session.current.use_primary! unless all_caught_up?(namespace, id) + end + + # Select a replica that has caught up with the primary. If one has not been + # found, stick to the primary. + def self.select_valid_host(namespace, id) + replica_selected = select_caught_up_replicas(namespace, id) + + Session.current.use_primary! unless replica_selected + end + + # Starts sticking to the primary for the given namespace and id, using + # the latest WAL pointer from the primary. + def self.stick(namespace, id) + return unless LoadBalancing.enable? + + mark_primary_write_location(namespace, id) + Session.current.use_primary! + end + + def self.bulk_stick(namespace, ids) + return unless LoadBalancing.enable? + + with_primary_write_location do |location| + ids.each do |id| + set_write_location_for(namespace, id, location) + end + end + + Session.current.use_primary! + end + + def self.with_primary_write_location + return unless LoadBalancing.configured? + + # Load balancing could be enabled for the Web application server, + # but it's not activated for Sidekiq. We should update Redis with + # the write location just in case load balancing is being used. + location = + if LoadBalancing.enable? + load_balancer.primary_write_location + else + Gitlab::Database.get_write_location(ActiveRecord::Base.connection) + end + + return if location.blank? + + yield(location) + end + + def self.mark_primary_write_location(namespace, id) + with_primary_write_location do |location| + set_write_location_for(namespace, id, location) + end + end + + # Stops sticking to the primary. + def self.unstick(namespace, id) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_key_for(namespace, id)) + end + end + + def self.set_write_location_for(namespace, id, location) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) + end + end + + def self.last_write_location_for(namespace, id) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_key_for(namespace, id)) + end + end + + def self.redis_key_for(namespace, id) + "database-load-balancing/write-location/#{namespace}/#{id}" + end + + def self.load_balancer + LoadBalancing.proxy.load_balancer + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3a94e109d2a..d155abefdc8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -5,7 +5,7 @@ module Gitlab module MigrationHelpers include Migrations::BackgroundMigrationHelpers include DynamicModelHelpers - include Migrations::RenameTableHelpers + include RenameTableHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -1091,6 +1091,25 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:) + migration = Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(job_class_name, table_name, column_name, job_arguments).first + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + if migration.nil? + Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" + elsif !migration.finished? + raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ + "but it is '#{migration.status}': #{configuration}" + end + end + # Returns an Array containing the indexes for the given column def indexes_for(table, column) column = column.to_s diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 8d5ea652bfc..fa30ffb62f5 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -131,12 +131,51 @@ module Gitlab final_delay end + # Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals + # + # This method is useful to schedule jobs that had previously failed. + # + # job_class_name - The background migration job class as a string + # delay_interval - The duration between each job's scheduled time + # batch_size - The maximum number of jobs to fetch to memory from the database. + def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max + + final_delay = 0 + job_counter = 0 + + jobs = Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: job_class_name) + jobs.each_batch(of: batch_size) do |job_batch| + job_batch.each do |job| + final_delay = initial_delay + delay_interval * job_counter + + migrate_in(final_delay, job_class_name, job.arguments) + + job_counter += 1 + end + end + + duration = initial_delay + delay_interval * job_counter + say <<~SAY + Scheduled #{job_counter} #{job_class_name} jobs with an interval of #{delay_interval} seconds. + + The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}." + SAY + + duration + end + # Creates a batched background migration for the given table. A batched migration runs one job # at a time, computing the bounds of the next batch based on the current migration settings and the previous # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be # present in the Gitlab::BackgroundMigration::BatchingStrategies module. # + # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper + # will log an warning and not create a new one. + # # job_class_name - The background migration job class as a string # batch_table_name - The name of the table the migration will batch over # batch_column_name - The name of the column the migration will batch over @@ -180,6 +219,13 @@ module Gitlab sub_batch_size: SUB_BATCH_SIZE ) + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ + "job_arguments: #{job_arguments.inspect}" + return + end + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY batch_max_value ||= connection.select_value(<<~SQL) @@ -194,13 +240,13 @@ module Gitlab job_class_name: job_class_name, table_name: batch_table_name, column_name: batch_column_name, + job_arguments: job_arguments, interval: job_interval, min_value: batch_min_value, max_value: batch_max_value, batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - job_arguments: job_arguments, status: migration_status) # This guard is necessary since #total_tuple_count was only introduced schema-wise, diff --git a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb index 906312478ac..88affaa9757 100644 --- a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb +++ b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This patch will be included in the next Rails release: https://github.com/rails/rails/pull/42368 +raise 'This patch can be removed' if Rails::VERSION::MAJOR > 6 + # rubocop:disable Gitlab/ModuleWithInstanceVariables module Gitlab module Database diff --git a/lib/gitlab/database/postgresql_adapter/type_map_cache.rb b/lib/gitlab/database/postgresql_adapter/type_map_cache.rb new file mode 100644 index 00000000000..ff66d9115ab --- /dev/null +++ b/lib/gitlab/database/postgresql_adapter/type_map_cache.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Caches loading of additional types from the DB +# https://github.com/rails/rails/blob/v6.0.3.2/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L521-L589 + +# rubocop:disable Gitlab/ModuleWithInstanceVariables + +module Gitlab + module Database + module PostgresqlAdapter + module TypeMapCache + extend ActiveSupport::Concern + + TYPE_MAP_CACHE_MONITOR = ::Monitor.new + + class_methods do + def type_map_cache + TYPE_MAP_CACHE_MONITOR.synchronize do + @type_map_cache ||= {} + end + end + end + + def initialize_type_map(map = type_map) + TYPE_MAP_CACHE_MONITOR.synchronize do + cached_type_map = self.class.type_map_cache[@connection_parameters.hash] + break @type_map = cached_type_map if cached_type_map + + super + self.class.type_map_cache[@connection_parameters.hash] = map + end + end + + def reload_type_map + TYPE_MAP_CACHE_MONITOR.synchronize do + self.class.type_map_cache[@connection_parameters.hash] = nil + end + + super + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 9ed03c05f0b..f3f0f227a8c 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -19,6 +19,7 @@ module Gitlab @diffable = diffable @include_stats = diff_options.delete(:include_stats) + @pagination_data = diff_options.delete(:pagination_data) @project = project @diff_options = diff_options @diff_refs = diff_refs @@ -47,11 +48,7 @@ module Gitlab end def pagination_data - { - current_page: nil, - next_page: nil, - total_pages: nil - } + @pagination_data || empty_pagination_data end # This mutates `diff_files` lines. @@ -90,6 +87,14 @@ module Gitlab private + def empty_pagination_data + { + current_page: nil, + next_page: nil, + total_pages: nil + } + end + def diff_stats_collection strong_memoize(:diff_stats) do next unless fetch_diff_stats? diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 64523f3b730..5ff7c88970c 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -21,9 +21,9 @@ module Gitlab @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) @pagination_data = { - current_page: batch_gradual_load? ? nil : @paginated_collection.current_page, - next_page: batch_gradual_load? ? nil : @paginated_collection.next_page, - total_pages: batch_gradual_load? ? relation.size : @paginated_collection.total_pages + current_page: current_page, + next_page: next_page, + total_pages: total_pages } end @@ -62,6 +62,24 @@ module Gitlab @merge_request_diff.merge_request_diff_files end + def current_page + return if @paginated_collection.blank? + + batch_gradual_load? ? nil : @paginated_collection.current_page + end + + def next_page + return if @paginated_collection.blank? + + batch_gradual_load? ? nil : @paginated_collection.next_page + end + + def total_pages + return if @paginated_collection.blank? + + batch_gradual_load? ? relation.size : @paginated_collection.total_pages + end + # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 6a41ed0f29e..32ce35110f8 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -70,12 +70,6 @@ module Gitlab return rich_line if marker_ranges.blank? begin - # MarkerRange objects are converted to Ranges to keep the previous behavior - # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 - if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) - marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range } - end - InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges) # This should only happen when the encoding of the diff doesn't # match the blob, which is a bug. But we shouldn't fail to render diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 209462fd6e9..a792eafde79 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -74,7 +74,6 @@ module Gitlab diffable.cache_key, VERSION, diff_options, - Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) ].join(":") diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 63334169c8e..fd3143488b1 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -84,6 +84,8 @@ module Gitlab end def valid_project_slug?(found_project) + return false unless found_project + project_slug == found_project.full_path_slug end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index cab3538a447..05daa08530e 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -65,10 +65,9 @@ module Gitlab def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) - project = Project.find_by_service_desk_project_key(match[:key]) - return unless valid_project_key?(project, match[:slug]) - - project + Project.with_service_desk_key(match[:key]).find do |project| + valid_project_key?(project, match[:slug]) + end end def valid_project_key?(project, slug) diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb index d538238f26f..fb4315e74b2 100644 --- a/lib/gitlab/email/message/in_product_marketing.rb +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -6,10 +6,8 @@ module Gitlab module InProductMarketing UnknownTrackError = Class.new(StandardError) - TRACKS = [:create, :verify, :team, :trial].freeze - def self.for(track) - raise UnknownTrackError unless TRACKS.include?(track) + raise UnknownTrackError unless Namespaces::InProductMarketingEmailsService::TRACKS.key?(track) "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 6341a7c7596..89acc058a46 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -10,10 +10,11 @@ module Gitlab attr_accessor :format - def initialize(group:, series:, format: :html) + def initialize(group:, user:, series:, format: :html) raise ArgumentError, "Only #{total_series} series available for this track." unless series.between?(0, total_series - 1) @group = group + @user = user @series = series @format = format end @@ -103,11 +104,7 @@ module Gitlab protected - attr_reader :group, :series - - def total_series - 3 - end + attr_reader :group, :user, :series private @@ -115,6 +112,10 @@ module Gitlab self.class.name.demodulize.downcase.to_sym end + def total_series + Namespaces::InProductMarketingEmailsService::TRACKS[track][:interval_days].size + end + def unsubscribe_com [ s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), diff --git a/lib/gitlab/email/message/in_product_marketing/experience.rb b/lib/gitlab/email/message/in_product_marketing/experience.rb new file mode 100644 index 00000000000..4156a737517 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/experience.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Experience < Base + include Gitlab::Utils::StrongMemoize + + EASE_SCORE_SURVEY_ID = 1 + + def subject_line + s_('InProductMarketing|Do you have a minute?') + end + + def tagline + end + + def title + s_('InProductMarketing|We want your GitLab experience to be great') + end + + def subtitle + s_('InProductMarketing|Take this 1-question survey!') + end + + def body_line1 + s_('InProductMarketing|%{strong_start}Overall, how difficult or easy was it to get started with GitLab?%{strong_end}').html_safe % strong_options + end + + def body_line2 + s_('InProductMarketing|Click on the number below that corresponds with your answer — 1 being very difficult, 5 being very easy.') + end + + def cta_text + end + + def feedback_link(rating) + params = { + onboarding_progress: onboarding_progress, + response: rating, + show_invite_link: show_invite_link, + survey_id: EASE_SCORE_SURVEY_ID + } + + "#{Gitlab::Saas.com_url}/-/survey_responses?#{params.to_query}" + end + + def feedback_ratings(rating) + [ + s_('InProductMarketing|Very difficult'), + s_('InProductMarketing|Difficult'), + s_('InProductMarketing|Neutral'), + s_('InProductMarketing|Easy'), + s_('InProductMarketing|Very easy') + ][rating - 1] + end + + def feedback_thanks + s_('InProductMarketing|Feedback from users like you really improves our product. Thanks for your help!') + end + + private + + def onboarding_progress + strong_memoize(:onboarding_progress) do + group.onboarding_progress.number_of_completed_actions + end + end + + def show_invite_link + strong_memoize(:show_invite_link) do + group.member_count > 1 && group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en' + end + end + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 71db8ab6067..8139a294269 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -20,7 +20,7 @@ module Gitlab raise UnknownIncomingEmail unless handler handler.execute.tap do - Gitlab::Metrics.add_event(handler.metrics_event, handler.metrics_params) + Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index e6f71e3ad3c..2b5f465d3c5 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -41,7 +41,17 @@ module Gitlab end def emoji_image_tag(name, src) - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />" + image_options = { + class: 'emoji', + src: src, + title: ":#{name}:", + alt: ":#{name}:", + height: 20, + width: 20, + align: 'absmiddle' + } + + ActionController::Base.helpers.tag(:img, image_options) end def emoji_exists?(name) diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index e91488c7c27..38ac5d9af74 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -146,9 +146,6 @@ module Gitlab else inject_context_for_exception(event, ex.cause) if ex.cause.present? end - # This should only happen on PostgreSQL v12 queries - rescue PgQuery::ParseError - event.extra[:sql] = ex.sql.to_s end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 8c916375a98..d5bf0cffb1e 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -67,10 +67,11 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) + Gitlab::ApplicationContext.push(feature_category: route.feature_category) + new_headers = { 'ETag' => etag, - 'X-Gitlab-From-Cache' => 'true', - ::Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER => route.feature_category + 'X-Gitlab-From-Cache' => 'true' } [status_code, new_headers, []] diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index da5b0afad38..7cf0232fbf2 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -25,7 +25,7 @@ module Gitlab # a proc that computes the sleep time given the number of preceding attempts # (from 1 to retries - 1) # - # Note: It's basically discouraged to use this method in a unicorn thread, + # Note: It's basically discouraged to use this method in a webserver thread, # because this ties up all thread related resources until all `retries` are consumed. # This could potentially eat up all connection pools. def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index e4233b8a935..fe3dd4759d6 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -45,12 +45,6 @@ module Gitlab remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, - invite_members_empty_project_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' - }, - trial_during_signup: { - tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' - }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, @@ -62,10 +56,12 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' }, learn_gitlab_a: { - tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA' + tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA', + rollout_strategy: :user }, learn_gitlab_b: { - tracking_category: 'Growth::Activation::Experiment::LearnGitLabB' + tracking_category: 'Growth::Activation::Experiment::LearnGitLabB', + rollout_strategy: :user }, in_product_marketing_emails: { tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index e53689eb89b..ca9205a8f8c 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -56,7 +56,7 @@ module Gitlab return if dnt_enabled? track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user)) end end diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index e398a3f9585..a8719761278 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -11,7 +11,7 @@ module Gitlab end def self.dir_glob - Dir.glob([Rails.root.join('file_hooks/*'), Rails.root.join('plugins/*')]) + Dir.glob(Rails.root.join('file_hooks/*')) end private_class_method :dir_glob diff --git a/lib/gitlab/file_hook_logger.rb b/lib/gitlab/file_hook_logger.rb index c5e69172016..4d6a650161f 100644 --- a/lib/gitlab/file_hook_logger.rb +++ b/lib/gitlab/file_hook_logger.rb @@ -3,7 +3,7 @@ module Gitlab class FileHookLogger < Gitlab::Logger def self.file_name_noext - 'plugin' + 'file_hook' end end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 751184b23df..aa5d50d1fb1 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -18,9 +18,9 @@ module Gitlab def conflicts @conflicts ||= wrapped_gitaly_errors do gitaly_conflicts_client(@target_repository).list_conflict_files.to_a + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message end - rescue GRPC::FailedPrecondition => e - raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError, e end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index fb947c80b7e..631624c068c 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -12,11 +12,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits def self.default_limits(project: nil) - if Feature.enabled?(:increased_diff_limits, project) - { max_files: 300, max_lines: 10000 } - else - { max_files: 100, max_lines: 5000 } - end + { max_files: ::Commit.diff_safe_max_files(project: project), max_lines: ::Commit.diff_safe_max_lines(project: project) } end def self.limits(options = {}) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index a8d1ea08275..6c4191ce25b 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -3,13 +3,13 @@ module Gitlab module Git class LfsChanges - def initialize(repository, newrev = nil) + def initialize(repository, newrevs = nil) @repository = repository - @newrev = newrev + @newrevs = newrevs end def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil) - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout) + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrevs, object_limit, not_in, dynamic_timeout) end def all_pointers diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 234541d8145..0ea009930b0 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -53,23 +53,6 @@ module Gitlab gitaly_repository.relative_path == other_repository.relative_path end - def fetch_env - gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) - gitaly_address = gitaly_client.address(storage) - gitaly_token = gitaly_client.token(storage) - - request = Gitaly::SSHUploadPackRequest.new(repository: gitaly_repository) - env = { - 'GITALY_ADDRESS' => gitaly_address, - 'GITALY_PAYLOAD' => request.to_json, - 'GITALY_WD' => Dir.pwd, - 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" - } - env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? - - env - end - def path @repository.path end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 102fe60f2cb..e38c7b516ee 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -302,8 +302,6 @@ module Gitlab private :archive_file_path def archive_version_path - return '' unless Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) - '@v2' end private :archive_version_path @@ -797,15 +795,19 @@ module Gitlab # Fetch remote for repository # # remote - remote name + # url - URL of the remote to fetch. `remote` is not used in this case. + # refmap - if url is given, determines which references should get fetched where # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # prune - should we use --prune flag? # check_tags_changed - should we ask gitaly to calculate whether any tags changed? - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) + def fetch_remote(remote, url: nil, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) wrapped_gitaly_errors do gitaly_repository_client.fetch_remote( remote, + url: url, + refmap: refmap, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b5e7220889e..b2a65d9f2d8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -334,23 +334,15 @@ module Gitlab # clear stale lock files. project.repository.clean_stale_repository_files if project.present? - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each.with_index do |change, index| - first_change = index == 0 - - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - check_single_change_access(change, skip_lfs_integrity_check: !first_change) - end + check_access! end end - def check_single_change_access(change, skip_lfs_integrity_check: false) - Checks::ChangeAccess.new( - change, + def check_access! + Checks::ChangesAccess.new( + changes_list.changes, user_access: user_access, project: project, - skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol, logger: logger ).validate! diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 9a431dc7088..4d87b91764a 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -109,20 +109,18 @@ module Gitlab end check_size_before_push! + check_access! + check_push_size! + end + override :check_access! + def check_access! changes_list.each do |change| # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up - check_single_change_access(change) + Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, root_ref: snippet.repository.root_ref, logger: logger).validate! + Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate! end - - check_push_size! - end - - override :check_single_change_access - def check_single_change_access(change, _skip_lfs_integrity_check: false) - Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, root_ref: snippet.repository.root_ref, logger: logger).validate! - Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index affd3986381..e4c8dc150a5 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -77,8 +77,8 @@ module Gitlab map_blob_types(response) end - def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) - request, rpc = create_new_lfs_pointers_request(revision, limit, not_in) + def get_new_lfs_pointers(revisions, limit, not_in, dynamic_timeout = nil) + request, rpc = create_new_lfs_pointers_request(revisions, limit, not_in) timeout = if dynamic_timeout @@ -109,7 +109,7 @@ module Gitlab private - def create_new_lfs_pointers_request(revision, limit, not_in) + def create_new_lfs_pointers_request(revisions, limit, not_in) # If the check happens for a change which is using a quarantine # environment for incoming objects, then we can avoid doing the # necessary graph walk to detect only new LFS pointers and instead scan @@ -126,7 +126,7 @@ module Gitlab [request, :list_all_lfs_pointers] else - revisions = [revision] + revisions = Array.wrap(revisions) revisions += if not_in.nil? || not_in == :all ["--not", "--all"] else diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 04dd394a2bd..1f360385111 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -45,18 +45,9 @@ module Gitlab # The remote_name parameter is deprecated and will be removed soon. def find_remote_root_ref(remote_name, remote_url, authorization) - request = if Feature.enabled?(:find_remote_root_refs_inmemory, default_enabled: :yaml) - Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote_url: remote_url, - http_authorization_header: authorization - ) - else - Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote: remote_name - ) - end + request = Gitaly::FindRemoteRootRefRequest.new(repository: @gitaly_repo, + remote_url: remote_url, + http_authorization_header: authorization) response = GitalyClient.call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d2dbd456180..6a75096ff80 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -70,13 +70,21 @@ module Gitlab end.join end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) + # rubocop: disable Metrics/ParameterLists + # The `remote` parameter is going away soonish anyway, at which point the + # Rubocop warning can be enabled again. + def fetch_remote(remote, url:, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags, timeout: timeout, no_prune: !prune, check_tags_changed: check_tags_changed ) + if url + request.remote_params = Gitaly::Remote.new(url: url, + mirror_refmaps: Array.wrap(refmap).map(&:to_s)) + end + if ssh_auth&.ssh_mirror_url? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? request.ssh_key = ssh_auth.ssh_private_key @@ -89,6 +97,7 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) end + # rubocop: enable Metrics/ParameterLists def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 7f1569f592f..28cd3f802a2 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -36,7 +36,11 @@ module Gitlab # updating the timestamp. project.update_column(:last_repository_updated_at, Time.zone.now) - project.repository.fetch_remote('github', forced: false) + if Feature.enabled?(:fetch_remote_params, project, default_enabled: :yaml) + project.repository.fetch_remote('github', url: project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false) + else + project.repository.fetch_remote('github', forced: false) + end pname = project.path_with_namespace diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 827027203ff..809a518d13a 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -6,6 +6,13 @@ module Gitlab class PullRequestsReviewsImporter include ParallelScheduling + def initialize(...) + super + + @merge_requests_already_imported_cache_key = + "github-importer/merge_request/already-imported/#{project.id}" + end + def importer_class PullRequestReviewImporter end @@ -22,11 +29,31 @@ module Gitlab :pull_request_reviews end - def id_for_already_imported_cache(merge_request) - merge_request.id + def id_for_already_imported_cache(review) + review.id + end + + def each_object_to_import(&block) + if use_github_review_importer_query_only_unimported_merge_requests? + each_merge_request_to_import(&block) + else + each_merge_request_skipping_imported(&block) + end end - def each_object_to_import + private + + attr_reader :merge_requests_already_imported_cache_key + + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62036#note_587181108 + def use_github_review_importer_query_only_unimported_merge_requests? + Feature.enabled?( + :github_review_importer_query_only_unimported_merge_requests, + default_enabled: :yaml + ) + end + + def each_merge_request_skipping_imported project.merge_requests.find_each do |merge_request| next if already_imported?(merge_request) @@ -40,6 +67,67 @@ module Gitlab mark_as_imported(merge_request) end end + + # The worker can be interrupted, by rate limit for instance, + # in different situations. To avoid requesting already imported data, + # if the worker is interrupted: + # - before importing all reviews of a merge request + # The reviews page is cached with the `PageCounter`, by merge request. + # - before importing all merge requests reviews + # Merge requests that had all the reviews imported are cached with + # `mark_merge_request_reviews_imported` + def each_merge_request_to_import + each_review_page do |page, merge_request| + page.objects.each do |review| + next if already_imported?(review) + + review.merge_request_id = merge_request.id + yield(review) + + mark_as_imported(review) + end + end + end + + def each_review_page + merge_requests_to_import.find_each do |merge_request| + # The page counter needs to be scoped by merge request to avoid skipping + # pages of reviews from already imported merge requests. + page_counter = PageCounter.new(project, page_counter_id(merge_request)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, merge_request.iid, options) do |page| + next unless page_counter.set(page.number) + + yield(page, merge_request) + end + + # Avoid unnecessary Redis cache keys after the work is done. + page_counter.expire! + mark_merge_request_reviews_imported(merge_request) + end + end + + # Returns only the merge requests that still have reviews to be imported. + def merge_requests_to_import + project.merge_requests.where.not(id: already_imported_merge_requests) # rubocop: disable CodeReuse/ActiveRecord + end + + def already_imported_merge_requests + Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key) + end + + def page_counter_id(merge_request) + "merge_request/#{merge_request.id}/#{collection_method}" + end + + def mark_merge_request_reviews_imported(merge_request) + Gitlab::Cache::Import::Caching.set_add( + merge_requests_already_imported_cache_key, + merge_request.id + ) + end end end end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index 3b4fd42ba2a..3face4c794b 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -26,6 +26,10 @@ module Gitlab def current Gitlab::Cache::Import::Caching.read_integer(cache_key) || 1 end + + def expire! + Gitlab::Cache::Import::Caching.expire(cache_key, 0) + end end end end diff --git a/lib/gitlab/global_id/deprecations.rb b/lib/gitlab/global_id/deprecations.rb new file mode 100644 index 00000000000..ac4a44e0e10 --- /dev/null +++ b/lib/gitlab/global_id/deprecations.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module GlobalId + module Deprecations + Deprecation = Struct.new(:old_model_name, :new_model_name, :milestone, keyword_init: true) + + # Contains the deprecations in place. + # Example: + # + # DEPRECATIONS = [ + # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.0') + # ].freeze + DEPRECATIONS = [ + # This works around an accidentally released argument named as `"EEIterationID"` in 7000489db. + Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3') + ].freeze + + # Maps of the DEPRECATIONS Hash for quick access. + OLD_NAME_MAP = DEPRECATIONS.index_by(&:old_model_name).freeze + NEW_NAME_MAP = DEPRECATIONS.index_by(&:new_model_name).freeze + OLD_GRAPHQL_NAME_MAP = DEPRECATIONS.index_by do |d| + Types::GlobalIDType.model_name_to_graphql_name(d.old_model_name) + end.freeze + + def self.deprecated?(old_model_name) + OLD_NAME_MAP.key?(old_model_name) + end + + def self.deprecation_for(old_model_name) + OLD_NAME_MAP[old_model_name] + end + + def self.deprecation_by(new_model_name) + NEW_NAME_MAP[new_model_name] + end + + # Returns the new `graphql_name` (Type#graphql_name) of a deprecated GID, + # or the `graphql_name` argument given if no deprecation applies. + def self.apply_to_graphql_name(graphql_name) + return graphql_name unless deprecation = OLD_GRAPHQL_NAME_MAP[graphql_name] + + Types::GlobalIDType.model_name_to_graphql_name(deprecation.new_model_name) + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 1fd210c521e..14f9c7f2191 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -47,6 +47,7 @@ module Gitlab push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) + push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb deleted file mode 100644 index 74c04e5380e..00000000000 --- a/lib/gitlab/graphql.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - StandardGraphqlError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 4d575b964e5..dc49c806398 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -51,14 +51,11 @@ module Gitlab object end - # authorizes the object using the current class authorization. def authorize!(object) raise_resource_not_available_error! unless authorized_resource?(object) end def authorized_resource?(object) - # Sanity check. We don't want to accidentally allow a developer to authorize - # without first adding permissions to authorize against raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none? self.class.authorization.ok?(object, current_user) diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index 8b73eeb4e52..20068758502 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -41,7 +41,7 @@ module Gitlab parts = [ "#{deprecated_in(format: :markdown)}.", reason_text, - replacement.then { |r| "Use: [`#{r}`](##{r.downcase.tr('.', '')})." if r } + replacement_markdown.then { |r| "Use: #{r}." if r } ].compact case context @@ -52,6 +52,13 @@ module Gitlab end end + def replacement_markdown + return unless replacement.present? + return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked + + "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" + end + def edit_description(original_description) @original_description = original_description return unless original_description diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb deleted file mode 100644 index b598b605141..00000000000 --- a/lib/gitlab/graphql/docs/helper.rb +++ /dev/null @@ -1,434 +0,0 @@ -# frozen_string_literal: true - -return if Rails.env.production? - -module Gitlab - module Graphql - module Docs - # We assume a few things about the schema. We use the graphql-ruby gem, which enforces: - # - All mutations have a single input field named 'input' - # - All mutations have a payload type, named after themselves - # - All mutations have an input type, named after themselves - # If these things change, then some of this code will break. Such places - # are guarded with an assertion that our assumptions are not violated. - ViolatedAssumption = Class.new(StandardError) - - SUGGESTED_ACTION = <<~MSG - We expect it to be impossible to violate our assumptions about - how mutation arguments work. - - If that is not the case, then something has probably changed in the - way we generate our schema, perhaps in the library we use: graphql-ruby - - Please ask for help in the #f_graphql or #backend channels. - MSG - - CONNECTION_ARGS = %w[after before first last].to_set - - FIELD_HEADER = <<~MD - #### Fields - - | Name | Type | Description | - | ---- | ---- | ----------- | - MD - - ARG_HEADER = <<~MD - # Arguments - - | Name | Type | Description | - | ---- | ---- | ----------- | - MD - - CONNECTION_NOTE = <<~MD - This field returns a [connection](#connections). It accepts the - four standard [pagination arguments](#connection-pagination-arguments): - `before: String`, `after: String`, `first: Int`, `last: Int`. - MD - - # Helper with functions to be used by HAML templates - # This includes graphql-docs gem helpers class. - # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb - module Helper - include GraphQLDocs::Helpers - include Gitlab::Utils::StrongMemoize - - def auto_generated_comment - <<-MD.strip_heredoc - --- - stage: Plan - group: Project Management - info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers - --- - - <!--- - This documentation is auto generated by a script. - - Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. - ---> - MD - end - - # Template methods: - # Methods that return chunks of Markdown for insertion into the document - - def render_full_field(field, heading_level: 3, owner: nil) - conn = connection?(field) - args = field[:arguments].reject { |arg| conn && CONNECTION_ARGS.include?(arg[:name]) } - arg_owner = [owner, field[:name]] - - chunks = [ - render_name_and_description(field, level: heading_level, owner: owner), - render_return_type(field), - render_input_type(field), - render_connection_note(field), - render_argument_table(heading_level, args, arg_owner), - render_return_fields(field, owner: owner) - ] - - join(:block, chunks) - end - - def render_argument_table(level, args, owner) - arg_header = ('#' * level) + ARG_HEADER - render_field_table(arg_header, args, owner) - end - - def render_name_and_description(object, owner: nil, level: 3) - content = [] - - heading = '#' * level - name = [owner, object[:name]].compact.join('.') - - content << "#{heading} `#{name}`" - content << render_description(object, owner, :block) - - join(:block, content) - end - - def render_object_fields(fields, owner:, level_bump: 0) - return if fields.blank? - - (with_args, no_args) = fields.partition { |f| args?(f) } - type_name = owner[:name] if owner - header_prefix = '#' * level_bump - sections = [ - render_simple_fields(no_args, type_name, header_prefix), - render_fields_with_arguments(with_args, type_name, header_prefix) - ] - - join(:block, sections) - end - - def render_enum_value(enum, value) - render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline)) - end - - def render_union_member(member) - "- [`#{member}`](##{member.downcase})" - end - - # QUERIES: - - # Methods that return parts of the schema, or related information: - - def connection_object_types - objects.select { |t| t[:is_edge] || t[:is_connection] } - end - - def object_types - objects.reject { |t| t[:is_edge] || t[:is_connection] || t[:is_payload] } - end - - def interfaces - graphql_interface_types.map { |t| t.merge(fields: t[:fields] + t[:connections]) } - end - - def fields_of(type_name) - graphql_operation_types - .find { |type| type[:name] == type_name } - .values_at(:fields, :connections) - .flatten - .then { |fields| sorted_by_name(fields) } - end - - # Place the arguments of the input types on the mutation itself. - # see: `#input_types` - this method must not call `#input_types` to avoid mutual recursion - def mutations - @mutations ||= sorted_by_name(graphql_mutation_types).map do |t| - inputs = t[:input_fields] - input = inputs.first - name = t[:name] - - assert!(inputs.one?, "Expected exactly 1 input field named #{name}. Found #{inputs.count} instead.") - assert!(input[:name] == 'input', "Expected the input of #{name} to be named 'input'") - - input_type_name = input[:type][:name] - input_type = graphql_input_object_types.find { |t| t[:name] == input_type_name } - assert!(input_type.present?, "Cannot find #{input_type_name} for #{name}.input") - - arguments = input_type[:input_fields] - seen_type!(input_type_name) - t.merge(arguments: arguments) - end - end - - # We assume that the mutations have been processed first, marking their - # inputs as `seen_type?` - def input_types - mutations # ensure that mutations have seen their inputs first - graphql_input_object_types.reject { |t| seen_type?(t[:name]) } - end - - # We ignore the built-in enum types, and sort values by name - def enums - graphql_enum_types - .reject { |type| type[:values].empty? } - .reject { |enum_type| enum_type[:name].start_with?('__') } - .map { |type| type.merge(values: sorted_by_name(type[:values])) } - end - - private # DO NOT CALL THESE METHODS IN TEMPLATES - - # Template methods - - def render_return_type(query) - return unless query[:type] # for example, mutations - - "Returns #{render_field_type(query[:type])}." - end - - def render_simple_fields(fields, type_name, header_prefix) - render_field_table(header_prefix + FIELD_HEADER, fields, type_name) - end - - def render_fields_with_arguments(fields, type_name, header_prefix) - return if fields.empty? - - level = 5 + header_prefix.length - sections = sorted_by_name(fields).map do |f| - render_full_field(f, heading_level: level, owner: type_name) - end - - <<~MD.chomp - #{header_prefix}#### Fields with arguments - - #{join(:block, sections)} - MD - end - - def render_field_table(header, fields, owner) - return if fields.empty? - - fields = sorted_by_name(fields) - header + join(:table, fields.map { |f| render_field(f, owner) }) - end - - def render_field(field, owner) - render_row( - render_name(field, owner), - render_field_type(field[:type]), - render_description(field, owner, :inline) - ) - end - - def render_return_fields(mutation, owner:) - fields = mutation[:return_fields] - return if fields.blank? - - name = owner.to_s + mutation[:name] - render_object_fields(fields, owner: { name: name }) - end - - def render_connection_note(field) - return unless connection?(field) - - CONNECTION_NOTE.chomp - end - - def render_row(*values) - "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" - end - - def render_name(object, owner = nil) - rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if deprecated?(object, owner) - - return rendered_name unless owner - - owner = Array.wrap(owner).join('') - id = (owner + object[:name]).downcase - - %(<a id="#{id}"></a>) + rendered_name - end - - # Returns the object description. If the object has been deprecated, - # the deprecation reason will be returned in place of the description. - def render_description(object, owner = nil, context = :block) - if deprecated?(object, owner) - render_deprecation(object, owner, context) - else - render_description_of(object, owner, context) - end - end - - def deprecated?(object, owner) - return true if object[:is_deprecated] # only populated for fields, not arguments! - - key = [*Array.wrap(owner), object[:name]].join('.') - deprecations.key?(key) - end - - def render_description_of(object, owner, context = nil) - desc = if object[:is_edge] - base = object[:name].chomp('Edge') - "The edge type for [`#{base}`](##{base.downcase})." - elsif object[:is_connection] - base = object[:name].chomp('Connection') - "The connection type for [`#{base}`](##{base.downcase})." - else - object[:description]&.strip - end - - return if desc.blank? - - desc += '.' unless desc.ends_with?('.') - see = doc_reference(object, owner) - desc += " #{see}" if see - desc += " (see [Connections](#connections))" if connection?(object) && context != :block - desc - end - - def doc_reference(object, owner) - field = schema_field(owner, object[:name]) if owner - return unless field - - ref = field.try(:doc_reference) - return if ref.blank? - - parts = ref.to_a.map do |(title, url)| - "[#{title.strip}](#{url.strip})" - end - - "See #{parts.join(', ')}." - end - - def render_deprecation(object, owner, context) - buff = [] - deprecation = schema_deprecation(owner, object[:name]) - - buff << (deprecation&.original_description || render_description_of(object, owner)) if context == :block - buff << if deprecation - deprecation.markdown(context: context) - else - "**Deprecated:** #{object[:deprecation_reason]}" - end - - join(context, buff) - end - - def render_field_type(type) - "[`#{type[:info]}`](##{type[:name].downcase})" - end - - def join(context, chunks) - chunks.compact! - return if chunks.blank? - - case context - when :block - chunks.join("\n\n") - when :inline - chunks.join(" ").squish.presence - when :table - chunks.join("\n") - end - end - - # Queries - - def sorted_by_name(objects) - return [] unless objects.present? - - objects.sort_by { |o| o[:name] } - end - - def connection?(field) - type_name = field.dig(:type, :name) - type_name.present? && type_name.ends_with?('Connection') - end - - # We are ignoring connections and built in types for now, - # they should be added when queries are generated. - def objects - strong_memoize(:objects) do - mutations = schema.mutation&.fields&.keys&.to_set || [] - - graphql_object_types - .reject { |object_type| object_type[:name]["__"] || object_type[:name] == 'Subscription' } # We ignore introspection and subscription types. - .map do |type| - name = type[:name] - type.merge( - is_edge: name.ends_with?('Edge'), - is_connection: name.ends_with?('Connection'), - is_payload: name.ends_with?('Payload') && mutations.include?(name.chomp('Payload').camelcase(:lower)), - fields: type[:fields] + type[:connections] - ) - end - end - end - - def args?(field) - args = field[:arguments] - return false if args.blank? - return true unless connection?(field) - - args.any? { |arg| CONNECTION_ARGS.exclude?(arg[:name]) } - end - - # returns the deprecation information for a field or argument - # See: Gitlab::Graphql::Deprecation - def schema_deprecation(type_name, field_name) - key = [*Array.wrap(type_name), field_name].join('.') - deprecations[key] - end - - def render_input_type(query) - input_field = query[:input_fields]&.first - return unless input_field - - "Input type: `#{input_field[:type][:name]}`" - end - - def schema_field(type_name, field_name) - type = schema.types[type_name] - return unless type && type.kind.fields? - - type.fields[field_name] - end - - def deprecations - strong_memoize(:deprecations) do - mapping = {} - - schema.types.each do |type_name, type| - next unless type.kind.fields? - - type.fields.each do |field_name, field| - mapping["#{type_name}.#{field_name}"] = field.try(:deprecation) - field.arguments.each do |arg_name, arg| - mapping["#{type_name}.#{field_name}.#{arg_name}"] = arg.try(:deprecation) - end - end - end - - mapping.compact - end - end - - def assert!(claim, message) - raise ViolatedAssumption, "#{message}\n#{SUGGESTED_ACTION}" unless claim - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb deleted file mode 100644 index ae0898e6198..00000000000 --- a/lib/gitlab/graphql/docs/renderer.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -return if Rails.env.production? - -module Gitlab - module Graphql - module Docs - # Gitlab renderer for graphql-docs. - # Uses HAML templates to parse markdown and generate .md files. - # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. - # - # Arguments: - # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema - # output_dir: The folder where the markdown files will be saved - # template: The path of the haml template to be parsed - class Renderer - include Gitlab::Graphql::Docs::Helper - - attr_reader :schema - - def initialize(schema, output_dir:, template:) - @output_dir = output_dir - @template = template - @layout = Haml::Engine.new(File.read(template)) - @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse - @schema = schema - @seen = Set.new - end - - def contents - # Render and remove an extra trailing new line - @contents ||= @layout.render(self).sub!(/\n(?=\Z)/, '') - end - - def write - filename = File.join(@output_dir, 'index.md') - - FileUtils.mkdir_p(@output_dir) - File.write(filename, contents) - end - - private - - def seen_type?(name) - @seen.include?(name) - end - - def seen_type!(name) - @seen << name - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml deleted file mode 100644 index 7d42fb3a9f8..00000000000 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ /dev/null @@ -1,224 +0,0 @@ --# haml-lint:disable UnnecessaryStringOutput - -= auto_generated_comment - -:plain - # GraphQL API Resources - - This documentation is self-generated based on GitLab current GraphQL schema. - - The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). - - Each table below documents a GraphQL type. Types match loosely to models, but not all - fields and methods on a model are available via GraphQL. - - WARNING: - Fields that are deprecated are marked with **{warning-solid}**. - Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found - in [Removed Items](../removed_items.md). - - <!-- vale off --> - <!-- Docs linting disabled after this line. --> - <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> -\ - -:plain - ## `Query` type - - The `Query` type contains the API's top-level entry points for all executable queries. -\ - -- fields_of('Query').each do |field| - = render_full_field(field, heading_level: 3, owner: 'Query') - \ - -:plain - ## `Mutation` type - - The `Mutation` type contains all the mutations you can execute. - - All mutations receive their arguments in a single input object named `input`, and all mutations - support at least a return field `errors` containing a list of error messages. - - All input objects may have a `clientMutationId: String` field, identifying the mutation. - - For example: - - ```graphql - mutation($id: NoteableID!, $body: String!) { - createNote(input: { noteableId: $id, body: $body }) { - errors - } - } - ``` -\ - -- mutations.each do |field| - = render_full_field(field, heading_level: 3, owner: 'Mutation') - \ - -:plain - ## Connections - - Some types in our schema are `Connection` types - they represent a paginated - collection of edges between two nodes in the graph. These follow the - [Relay cursor connections specification](https://relay.dev/graphql/connections.htm). - - ### Pagination arguments {#connection-pagination-arguments} - - All connection fields support the following pagination arguments: - - | Name | Type | Description | - |------|------|-------------| - | `after` | [`String`](#string) | Returns the elements in the list that come after the specified cursor. | - | `before` | [`String`](#string) | Returns the elements in the list that come before the specified cursor. | - | `first` | [`Int`](#int) | Returns the first _n_ elements from the list. | - | `last` | [`Int`](#int) | Returns the last _n_ elements from the list. | - - Since these arguments are common to all connection fields, they are not repeated for each connection. - - ### Connection fields - - All connections have at least the following fields: - - | Name | Type | Description | - |------|------|-------------| - | `pageInfo` | [`PageInfo!`](#pageinfo) | Pagination information. | - | `edges` | `[edge!]` | The edges. | - | `nodes` | `[item!]` | The items in the current page. | - - The precise type of `Edge` and `Item` depends on the kind of connection. A - [`ProjectConnection`](#projectconnection) will have nodes that have the type - [`[Project!]`](#project), and edges that have the type [`ProjectEdge`](#projectedge). - - ### Connection types - - Some of the types in the schema exist solely to model connections. Each connection - has a distinct, named type, with a distinct named edge type. These are listed separately - below. -\ - -- connection_object_types.each do |type| - = render_name_and_description(type, level: 4) - \ - = render_object_fields(type[:fields], owner: type, level_bump: 1) - \ - -:plain - ## Object types - - Object types represent the resources that the GitLab GraphQL API can return. - They contain _fields_. Each field has its own type, which will either be one of the - basic GraphQL [scalar types](https://graphql.org/learn/schema/#scalar-types) - (e.g.: `String` or `Boolean`) or other object types. Fields may have arguments. - Fields with arguments are exactly like top-level queries, and are listed beneath - the table of fields for each object type. - - For more information, see - [Object Types and Fields](https://graphql.org/learn/schema/#object-types-and-fields) - on `graphql.org`. -\ - -- object_types.each do |type| - = render_name_and_description(type) - \ - = render_object_fields(type[:fields], owner: type) - \ - -:plain - ## Enumeration types - - Also called _Enums_, enumeration types are a special kind of scalar that - is restricted to a particular set of allowed values. - - For more information, see - [Enumeration Types](https://graphql.org/learn/schema/#enumeration-types) - on `graphql.org`. -\ - -- enums.each do |enum| - = render_name_and_description(enum) - \ - ~ "| Value | Description |" - ~ "| ----- | ----------- |" - - enum[:values].each do |value| - = render_enum_value(enum, value) - \ - -:plain - ## Scalar types - - Scalar values are atomic values, and do not have fields of their own. - Basic scalars include strings, boolean values, and numbers. This schema also - defines various custom scalar values, such as types for times and dates. - - This schema includes custom scalar types for identifiers, with a specific type for - each kind of object. - - For more information, read about [Scalar Types](https://graphql.org/learn/schema/#scalar-types) on `graphql.org`. -\ - -- graphql_scalar_types.each do |type| - = render_name_and_description(type) - \ - -:plain - ## Abstract types - - Abstract types (unions and interfaces) are ways the schema can represent - values that may be one of several concrete types. - - - A [`Union`](https://graphql.org/learn/schema/#union-types) is a set of possible types. - The types might not have any fields in common. - - An [`Interface`](https://graphql.org/learn/schema/#interfaces) is a defined set of fields. - Types may `implement` an interface, which - guarantees that they have all the fields in the set. A type may implement more than - one interface. - - See the [GraphQL documentation](https://graphql.org/learn/) for more information on using - abstract types. -\ - -:plain - ### Unions -\ - -- graphql_union_types.each do |type| - = render_name_and_description(type, level: 4) - \ - One of: - \ - - type[:possible_types].each do |member| - = render_union_member(member) - \ - -:plain - ### Interfaces -\ - -- interfaces.each do |type| - = render_name_and_description(type, level: 4) - \ - Implementations: - \ - - type[:implemented_by].each do |type_name| - ~ "- [`#{type_name}`](##{type_name.downcase})" - \ - = render_object_fields(type[:fields], owner: type, level_bump: 1) - \ - -:plain - ## Input types - - Types that may be used as arguments (all scalar types may also - be used as arguments). - - Only general use input types are listed here. For mutation input types, - see the associated mutation type above. -\ - -- input_types.each do |type| - = render_name_and_description(type) - \ - = render_argument_table(3, type[:input_fields], type[:name]) - \ diff --git a/lib/gitlab/graphql/standard_graphql_error.rb b/lib/gitlab/graphql/standard_graphql_error.rb new file mode 100644 index 00000000000..8364c232af2 --- /dev/null +++ b/lib/gitlab/graphql/standard_graphql_error.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# rubocop:disable Cop/CustomErrorClass + +module Gitlab + module Graphql + class StandardGraphqlError < StandardError + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index f7e46fce134..44b85bf886e 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -20,7 +20,8 @@ module Gitlab def check ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && - ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up && + ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up end end end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb new file mode 100644 index 00000000000..cf9fa700b0a --- /dev/null +++ b/lib/gitlab/health_checks/redis/trace_chunks_check.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + class TraceChunksCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_trace_chunks_ping' + end + + def successful?(result) + result == 'PONG' + end + + # rubocop: disable CodeReuse/ActiveRecord + def check + catch_timeout 10.seconds do + Gitlab::Redis::TraceChunks.with(&:ping) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb deleted file mode 100644 index f0c6fdab600..00000000000 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - # This check can only be run on Unicorn `master` process - class UnicornCheck - extend SimpleAbstractCheck - - class << self - include Gitlab::Utils::StrongMemoize - - private - - def metric_prefix - 'unicorn_check' - end - - def successful?(result) - result > 0 - end - - def check - return unless http_servers - - http_servers.sum(&:worker_processes) - end - - # Traversal of ObjectSpace is expensive, on fully loaded application - # it takes around 80ms. The instances of HttpServers are not a subject - # to change so we can cache the list of servers. - def http_servers - strong_memoize(:http_servers) do - next unless Gitlab::Runtime.unicorn? - - ObjectSpace.each_object(::Unicorn::HttpServer).to_a - end - end - end - end - end -end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index e4857280969..d05ced00a6b 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,9 +11,11 @@ module Gitlab end def self.too_large?(size) - return false unless size.to_i > Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + file_size_limit = Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] - over_highlight_size_limit.increment(source: "text highlighter") if Feature.enabled?(:track_file_size_over_highlight_limit) + return false unless size.to_i > file_size_limit + + over_highlight_size_limit.increment(source: "file size: #{file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) true end @@ -68,6 +70,8 @@ module Gitlab end def highlight_rich(text, continue: true) + add_highlight_attempt_metric + tag = lexer.tag tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } @@ -88,12 +92,25 @@ module Gitlab Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end + def add_highlight_attempt_metric + return unless Feature.enabled?(:track_highlight_timeouts) + + highlighting_attempt.increment(source: (@language || "undefined")) + end + def add_highlight_timeout_metric return unless Feature.enabled?(:track_highlight_timeouts) highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") end + def highlighting_attempt + @highlight_attempt ||= Gitlab::Metrics.counter( + :file_highlighting_attempt, + 'Counts the times highlighting has been attempted on a file' + ) + end + def highlight_timeout @highlight_timeout ||= Gitlab::Metrics.counter( :highlight_timeout, diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index d5595e80bdf..2d1bb515058 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,6 +7,7 @@ module Gitlab assignees labels total_time_spent + time_change ].freeze def self.safe_hook_attributes @@ -43,7 +44,9 @@ module Gitlab description: absolute_image_urls(issue.description), url: Gitlab::UrlBuilder.build(issue), total_time_spent: issue.total_time_spent, + time_change: issue.time_change, human_total_time_spent: issue.human_total_time_spent, + human_time_change: issue.human_time_change, human_time_estimate: issue.human_time_estimate, assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae2ec424ce5..db807a3c557 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -37,6 +37,7 @@ module Gitlab assignees labels total_time_spent + time_change ].freeze alias_method :merge_request, :object @@ -50,7 +51,9 @@ module Gitlab last_commit: merge_request.diff_head_commit&.hook_attrs, work_in_progress: merge_request.work_in_progress?, total_time_spent: merge_request.total_time_spent, + time_change: merge_request.time_change, human_total_time_spent: merge_request.human_total_time_spent, + human_time_change: merge_request.human_time_change, human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 023dbd1c601..30e72b58e21 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -40,24 +40,24 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 1, 'cs_CZ' => 1, - 'de' => 19, + 'de' => 18, 'en' => 100, 'eo' => 1, - 'es' => 41, + 'es' => 40, 'fil_PH' => 1, - 'fr' => 14, + 'fr' => 13, 'gl_ES' => 1, 'id_ID' => 0, 'it' => 2, - 'ja' => 45, - 'ko' => 14, + 'ja' => 44, + 'ko' => 13, 'nl_NL' => 1, - 'pl_PL' => 1, - 'pt_BR' => 22, - 'ru' => 32, + 'pl_PL' => 3, + 'pt_BR' => 21, + 'ru' => 30, 'tr_TR' => 17, - 'uk' => 43, - 'zh_CN' => 72, + 'uk' => 42, + 'zh_CN' => 69, 'zh_HK' => 3, 'zh_TW' => 4 }.freeze diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 1e8009d29c2..78608a946de 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -32,6 +32,10 @@ module Gitlab end end + def delete_export? + false + end + private def send_file diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 959ece4b903..30cd5ccfbcb 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -69,6 +69,7 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return @relation_hash if author_relation? return if invalid_relation? || predefined_relation? setup_base_models @@ -95,6 +96,10 @@ module Gitlab relation_class.try(:predefined_id?, @relation_hash['id']) end + def author_relation? + @relation_name == :author + end + def setup_models raise NotImplementedError end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index ace9d83dc9a..6c0b6de9e85 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,8 +15,17 @@ module Gitlab end def gzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename) + end + + def gunzip(dir:, filename:) + gzip_with_options(dir: dir, filename: filename, options: 'd') + end + + def gzip_with_options(dir:, filename:, options: nil) filepath = File.join(dir, filename) cmd = %W(gzip #{filepath}) + cmd << "-#{options}" if options _, status = Gitlab::Popen.popen(cmd) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 2baf2c61f7c..febfe00af0b 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -32,7 +32,16 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) stdin.close - pgrp = Process.getpgid(wait_thr[:pid]) + + # When validation is performed on a small archive (e.g. 100 bytes) + # `wait_thr` finishes before we can get process group id. Do not + # raise exception in this scenario. + pgrp = begin + Process.getpgid(wait_thr[:pid]) + rescue Errno::ESRCH + nil + end + status = wait_thr.value if status.success? diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 4af6b03fe94..af0026b8864 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -15,7 +15,7 @@ module Gitlab end def self.file_compression_error - self.new('File compression failed') + self.new('File compression/decompression failed') end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4b3258f8caa..5274fcec43e 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,9 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do - # Disable archive validation by default - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/235949 - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) decompress_archive end rescue StandardError => e diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index aceb4821a06..4786c7a52cc 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -70,11 +70,14 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji - events: - :push_event_payload + - :system_note_metadata - boards: - :board_assignee - :milestone diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 19611e1b010..0a6234f9f02 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -72,6 +72,8 @@ ee: - :award_emoji - events: - :push_event_payload + - label_links: + - :label - notes: - :author - :award_emoji diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 2b95c098b59..8b39362b6bb 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -55,11 +55,11 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do if @group_hash.present? - ImportExport::JSON::LegacyReader::Hash.new( + ImportExport::Json::LegacyReader::Hash.new( @group_hash, relation_names: reader.group_relation_names) else - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'group.json'), relation_names: reader.group_relation_names) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index ea7de4cc896..19d707aaca5 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -118,7 +118,7 @@ module Gitlab def relation_reader strong_memoize(:relation_reader) do - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 0f588a55f9d..796b9258e57 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -42,7 +42,7 @@ module Gitlab end def serialize(group) - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( group, group_tree, json_writer, @@ -64,7 +64,7 @@ module Gitlab end def json_writer - @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path) + @json_writer ||= ImportExport::Json::NdjsonWriter.new(@full_path) end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index f29c0a44188..97b34088e3e 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyReader class File < LegacyReader include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb index 7be21410d26..e03ab9f7650 100644 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class LegacyWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 5c8edd485e5..4899bd3b0ee 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonReader MAX_JSON_DOCUMENT_SIZE = 50.megabytes diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb index e74fdd74049..e303ac6eefa 100644 --- a/lib/gitlab/import_export/json/ndjson_writer.rb +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class NdjsonWriter include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index ec42c5e51c0..d1e013a151c 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - module JSON + module Json class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index f8b8b74ffd7..c6b961ea210 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab private def batch_size(exportable) - Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable) + Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) end end end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 113502b4e3c..d8992061524 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -56,13 +56,13 @@ module Gitlab def ndjson_relation_reader return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) - ImportExport::JSON::NdjsonReader.new( + ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end def legacy_relation_reader - ImportExport::JSON::LegacyReader::File.new( + ImportExport::Json::LegacyReader::File.new( File.join(shared.export_path, 'project.json'), relation_names: reader.project_relation_names, allowed_path: importable_path diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 16012f3c0c0..1f0fa249390 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -14,7 +14,7 @@ module Gitlab end def save - ImportExport::JSON::StreamingSerializer.new( + ImportExport::Json::StreamingSerializer.new( exportable, reader.project_tree, json_writer, @@ -56,10 +56,10 @@ module Gitlab @json_writer ||= begin if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) else full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project') + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index f295ab38de0..5cb1c1f8981 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -88,7 +88,7 @@ module Gitlab when 'Project' @exportable.disk_path when 'Group' - @exportable.full_path + Storage::Hashed.new(@exportable, prefix: Storage::Hashed::GROUP_REPOSITORY_PATH_PREFIX).disk_path else raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index d1ac6a55fb7..ab0e56adc32 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -8,8 +8,9 @@ module Gitlab Cache = Class.new(RedisBase).enable_redis_cluster_validation Queues = Class.new(RedisBase) SharedState = Class.new(RedisBase).enable_redis_cluster_validation + TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - STORAGES = [ActionCable, Cache, Queues, SharedState].freeze + STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze @@ -21,10 +22,6 @@ module Gitlab nil end - def known_payload_keys - super + STORAGES.flat_map(&:known_payload_keys) - end - def payload super.merge(*STORAGES.flat_map(&:payload)) end diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb index 69aafffd124..86a6525c8d0 100644 --- a/lib/gitlab/instrumentation/redis_payload.rb +++ b/lib/gitlab/instrumentation/redis_payload.rb @@ -5,12 +5,6 @@ module Gitlab module RedisPayload include ::Gitlab::Utils::StrongMemoize - # Fetches payload keys from the lazy payload (this avoids - # unnecessary processing of the values). - def known_payload_keys - to_lazy_payload.keys - end - def payload to_lazy_payload.transform_values do |value| result = value.call diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index e6ea98e6d66..9d7254f49f7 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -4,7 +4,10 @@ module Gitlab module Integrations class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( - Asana Assembla Bamboo Campfire Confluence Datadog EmailsOnPush + Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker + Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi Packagist PipelinesEmail Pivotaltracker + Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams )).freeze def cast(value) @@ -29,12 +32,16 @@ module Gitlab private + def namespaced_integrations + NAMESPACED_INTEGRATIONS + end + def new_cast(value) value = prepare_value(value) return unless value stripped_name = value.delete_suffix('Service') - return unless NAMESPACED_INTEGRATIONS.include?(stripped_name) + return unless namespaced_integrations.include?(stripped_name) "Integrations::#{stripped_name}" end @@ -55,3 +62,5 @@ module Gitlab end end end + +Gitlab::Integrations::StiType.prepend_mod diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 561cd4509b1..767ce310b5a 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -242,7 +242,7 @@ module Gitlab def self.encode(object, limit: 25.megabytes) return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder) - buffer = [] + buffer = StringIO.new buffer_size = 0 ::Yajl::Encoder.encode(object) do |data_chunk| @@ -254,7 +254,7 @@ module Gitlab buffer_size += chunk_size end - buffer.join('') + buffer.string end end end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 7b2c792ebca..a4663314b3b 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -45,6 +45,13 @@ module Gitlab Gitlab.config.gitlab_kas.external_url end + # Return GitLab KAS internal_url + # + # @return [String] internal_url + def internal_url + Gitlab.config.gitlab_kas.internal_url + end + # Return whether GitLab KAS is enabled # # @return [Boolean] external_url diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb new file mode 100644 index 00000000000..6675903e692 --- /dev/null +++ b/lib/gitlab/kas/client.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Kas + class Client + TIMEOUT = 2.seconds.freeze + JWT_AUDIENCE = 'gitlab-kas' + + STUB_CLASSES = { + configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub + }.freeze + + ConfigurationError = Class.new(StandardError) + + def initialize + raise ConfigurationError, 'GitLab KAS is not enabled' unless Gitlab::Kas.enabled? + raise ConfigurationError, 'KAS internal URL is not configured' unless Gitlab::Kas.internal_url.present? + end + + def list_agent_config_files(project:) + request = Gitlab::Agent::ConfigurationProject::Rpc::ListAgentConfigFilesRequest.new( + repository: repository(project), + gitaly_address: gitaly_address(project) + ) + + stub_for(:configuration_project) + .list_agent_config_files(request, metadata: metadata) + .config_files + .to_a + end + + private + + def stub_for(service) + @stubs ||= {} + @stubs[service] ||= STUB_CLASSES.fetch(service).new(kas_endpoint_url, credentials, timeout: TIMEOUT) + end + + def repository(project) + gitaly_repository = project.repository.gitaly_repository + + Gitlab::Agent::Modserver::Repository.new(gitaly_repository.to_h) + end + + def gitaly_address(project) + connection_data = Gitlab::GitalyClient.connection_data(project.repository_storage) + + Gitlab::Agent::Modserver::GitalyAddress.new(connection_data) + end + + def kas_endpoint_url + Gitlab::Kas.internal_url.delete_prefix('grpc://') + end + + def credentials + if Rails.env.test? || Rails.env.development? + :this_channel_is_insecure + else + GRPC::Core::ChannelCredentials.new + end + end + + def metadata + { 'authorization' => "bearer #{token}" } + end + + def token + JSONWebToken::HMACToken.new(Gitlab::Kas.secret).tap do |token| + token.issuer = Settings.gitlab.host + token.audience = JWT_AUDIENCE + end.encoded + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb deleted file mode 100644 index c5c5d198a6c..00000000000 --- a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module Parsers - # Parses Helm v2 list (JSON) output - class ListV2 - ParserError = Class.new(StandardError) - - attr_reader :contents, :json - - def initialize(contents) - @contents = contents - @json = Gitlab::Json.parse(contents) - rescue JSON::ParserError => e - raise ParserError, e.message - end - - def releases - @releases = helm_releases - end - - private - - def helm_releases - helm_releases = json['Releases'] || [] - - raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) } - - helm_releases - end - end - end - end - end -end diff --git a/lib/gitlab/markdown_cache/field_data.rb b/lib/gitlab/markdown_cache/field_data.rb index 14622c0f186..75364570640 100644 --- a/lib/gitlab/markdown_cache/field_data.rb +++ b/lib/gitlab/markdown_cache/field_data.rb @@ -9,7 +9,7 @@ module Gitlab @data = {} end - delegate :[], :[]=, to: :@data + delegate :[], :[]=, :key?, to: :@data def markdown_fields @data.keys diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 7bd55cce363..4c4942c12d5 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -16,6 +16,10 @@ module Gitlab @error end + def self.record_duration_for_status?(status) + status.to_i.between?(200, 499) + end + # Tracks an event. # # See `Gitlab::Metrics::Transaction#add_event` for more details. diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 558454eaa1c..756e6b0641a 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -30,8 +30,7 @@ module Gitlab # application: https://gitlab.com/gitlab-org/gitlab/issues/35343 self.readiness_checks = [ WebExporter::ExporterCheck.new(self), - Gitlab::HealthChecks::PumaCheck, - Gitlab::HealthChecks::UnicornCheck + Gitlab::HealthChecks::PumaCheck ] end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 19a835b9fc4..b99261b5c4d 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -15,7 +15,6 @@ module Gitlab HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze - FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' FEATURE_CATEGORY_DEFAULT = 'unknown' # These were the top 5 categories at a point in time, chosen as a @@ -67,18 +66,16 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase method = 'INVALID' unless HTTP_METHODS.key?(method) - started = Time.now.to_f + started = Gitlab::Metrics::System.monotonic_time health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' - feature_category = nil begin status, headers, body = @app.call(env) - elapsed = Time.now.to_f - started - feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil) + elapsed = Gitlab::Metrics::System.monotonic_time - started - unless health_endpoint + if !health_endpoint && Gitlab::Metrics.record_duration_for_status?(status) RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) end @@ -104,6 +101,10 @@ module Gitlab HEALTH_ENDPOINT.match?(CGI.unescape(path)) end + + def feature_category + ::Gitlab::ApplicationContext.current_context_attribute(:feature_category) + end end end end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 0a0ac6c5386..5d7f434b660 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -45,8 +45,8 @@ module Gitlab def labels_for_class(klass) { - host: klass.connection_config[:host], - port: klass.connection_config[:port], + host: klass.connection_db_config.host, + port: klass.connection_db_config.configuration_hash[:port], class: klass.to_s } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 3d29d38fa1f..b1c5e9800da 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client/support/unicorn' - module Gitlab module Metrics module Samplers diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb deleted file mode 100644 index 2fa324f3fea..00000000000 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Samplers - class UnicornSampler < BaseSampler - DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - - def metrics - @metrics ||= init_metrics - end - - def init_metrics - { - unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), - unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), - unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') - } - end - - def enabled? - # Raindrops::Linux.tcp_listener_stats is only present on Linux - unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) - end - - def sample - Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - set_unicorn_connection_metrics('tcp', addr, stats) - end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - set_unicorn_connection_metrics('unix', addr, stats) - end - - metrics[:unicorn_workers].set({}, unicorn_workers_count) - end - - private - - def tcp_listeners - @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) - end - - def set_unicorn_connection_metrics(type, addr, stats) - labels = { socket_type: type, socket_address: addr } - - metrics[:unicorn_active_connections].set(labels, stats.active) - metrics[:unicorn_queued_connections].set(labels, stats.queued) - end - - def unix_listeners - @unix_listeners ||= Unicorn.listener_names - tcp_listeners - end - - def unicorn_with_listeners? - defined?(Unicorn) && Unicorn.listener_names.any? - end - - def unicorn_workers_count - http_servers.sum(&:worker_processes) - end - - # Traversal of ObjectSpace is expensive, on fully loaded application - # it takes around 80ms. The instances of HttpServers are not a subject - # to change so we can cache the list of servers. - def http_servers - return [] unless Gitlab::Runtime.unicorn? - - @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a - end - end - end - end -end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 3db3317e833..9f7884e1364 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -14,6 +14,14 @@ module Gitlab SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze + DB_LOAD_BALANCING_COUNTERS = %i{ + db_replica_count db_replica_cached_count db_replica_wal_count + db_primary_count db_primary_cached_count db_primary_wal_count + }.freeze + DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze + + SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze + # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ActiveRecord::Base.transaction {} block. @@ -39,23 +47,56 @@ module Gitlab observe(:gitlab_sql_duration_seconds, event) do buckets SQL_DURATION_BUCKET end + + if ::Gitlab::Database::LoadBalancing.enable? + db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection]) + return if db_role.blank? + + increment_db_role_counters(db_role, payload) + observe_db_role_duration(db_role, event) + end end def self.db_counter_payload return {} unless Gitlab::SafeRequestStore.active? - payload = {} - DB_COUNTERS.each do |counter| - payload[counter] = Gitlab::SafeRequestStore[counter].to_i + {}.tap do |payload| + DB_COUNTERS.each do |counter| + payload[counter] = Gitlab::SafeRequestStore[counter].to_i + end + + if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + DB_LOAD_BALANCING_COUNTERS.each do |counter| + payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i + end + DB_LOAD_BALANCING_DURATIONS.each do |duration| + payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) + end + end end - payload end - def self.known_payload_keys - DB_COUNTERS + private + + def wal_command?(payload) + payload[:sql].match(SQL_WAL_LOCATION_REGEX) + end + + def increment_db_role_counters(db_role, payload) + increment("db_#{db_role}_count".to_sym) + increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload) + increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload) end - private + def observe_db_role_duration(db_role, event) + observe("gitlab_sql_#{db_role}_duration_seconds".to_sym, event) do + buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET + end + + duration = event.duration / 1000.0 + duration_key = "db_#{db_role}_duration_s".to_sym + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + end def ignored_query?(payload) payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) @@ -86,5 +127,3 @@ module Gitlab end end end - -Gitlab::Metrics::Subscribers::ActiveRecord.prepend_mod_with('Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 0df64f2897e..60a1b084345 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -14,8 +14,6 @@ module Gitlab COUNTER = :external_http_count DURATION = :external_http_duration_s - KNOWN_PAYLOAD_KEYS = [COUNTER, DURATION].freeze - def self.detail_store ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 3ebafb5c5e4..97cc8bed564 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -13,8 +13,6 @@ module Gitlab THREAD_KEY = :_gitlab_metrics_transaction - SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze - # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' @@ -39,29 +37,10 @@ module Gitlab def initialize @methods = {} - - @started_at = nil - @finished_at = nil - end - - def duration - @finished_at ? (@finished_at - @started_at) : 0.0 end def run - Thread.current[THREAD_KEY] = self - - @started_at = System.monotonic_time - - yield - ensure - @finished_at = System.monotonic_time - - observe(:gitlab_transaction_duration_seconds, duration) do - buckets SMALL_BUCKETS - end - - Thread.current[THREAD_KEY] = nil + raise NotImplementedError end # Tracks a business level event diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index ee9e6f449d3..3ebfcc43b0b 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -6,12 +6,29 @@ module Gitlab CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) + SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze def initialize(env) super() @env = env end + def run + Thread.current[THREAD_KEY] = self + + started_at = System.monotonic_time + + status, _, _ = retval = yield + + finished_at = System.monotonic_time + duration = finished_at - started_at + record_duration_if_needed(status, duration) + + retval + ensure + Thread.current[THREAD_KEY] = nil + end + def labels return @labels if @labels @@ -27,6 +44,14 @@ module Gitlab private + def record_duration_if_needed(status, duration) + return unless Gitlab::Metrics.record_duration_for_status?(status) + + observe(:gitlab_transaction_duration_seconds, duration) do + buckets SMALL_BUCKETS + end + end + def labels_from_controller controller = @env[CONTROLLER_KEY] diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index ee11f1f4560..4cb38e6bb9b 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -8,17 +8,17 @@ module Gitlab # this is already :/. We could also take a hash and manually check every # entry, but it's much more maintainable to do rely on native Ruby. # rubocop: disable Metrics/ParameterLists - def self.build(id:, title:, active: false, icon: '', href: '', method: nil, view: '', css_class: '', data: {}) + def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) { id: id, title: title, active: active, icon: icon, href: href, - method: method, view: view.to_s, css_class: css_class, - data: data + data: data || { qa_selector: 'menu_item_link', qa_title: title }, + emoji: emoji } end # rubocop: enable Metrics/ParameterLists diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 60f5b267071..11ca6a3a3ba 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -6,9 +6,34 @@ module Gitlab def initialize @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new @views = {} + @shortcuts = [] end - delegate :add_primary_menu_item, :add_secondary_menu_item, to: :@menu_builder + # Using delegate hides the stacktrace for some errors, so we choose to be explicit. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 + def add_primary_menu_item(**args) + @menu_builder.add_primary_menu_item(**args) + end + + def add_secondary_menu_item(**args) + @menu_builder.add_secondary_menu_item(**args) + end + + def add_shortcut(**args) + item = ::Gitlab::Nav::TopNavMenuItem.build(**args) + + @shortcuts.push(item) + end + + def add_primary_menu_item_with_shortcut(shortcut_class:, shortcut_href: nil, **args) + add_primary_menu_item(**args) + add_shortcut( + id: "#{args.fetch(:id)}-shortcut", + title: args.fetch(:title), + href: shortcut_href || args.fetch(:href), + css_class: shortcut_class + ) + end def add_view(name, props) @views[name] = props @@ -19,6 +44,7 @@ module Gitlab menu.merge({ views: @views, + shortcuts: @shortcuts, activeTitle: _('Menu') }) end diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb index 69c468207f6..888d93d5fe3 100644 --- a/lib/gitlab/pagination/keyset/header_builder.rb +++ b/lib/gitlab/pagination/keyset/header_builder.rb @@ -13,7 +13,6 @@ module Gitlab def add_next_page_header(query_params) link = next_page_link(page_href(query_params)) - header('Links', link) header('Link', link) end diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb new file mode 100644 index 00000000000..2ec4472fcd6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/paginator.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Paginator + include Enumerable + + module Base64CursorConverter + def self.dump(cursor_attributes) + Base64.urlsafe_encode64(Gitlab::Json.dump(cursor_attributes)) + end + + def self.parse(cursor) + Gitlab::Json.parse(Base64.urlsafe_decode64(cursor)).with_indifferent_access + end + end + + FORWARD_DIRECTION = 'n' + BACKWARD_DIRECTION = 'p' + + UnsupportedScopeOrder = Class.new(StandardError) + + # scope - ActiveRecord::Relation object with order by clause + # cursor - Encoded cursor attributes as String. Empty value will requests the first page. + # per_page - Number of items per page. + # cursor_converter - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods. + # direction_key - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction) + def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd) + @keyset_scope = build_scope(scope) + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope) + @per_page = per_page + @cursor_converter = cursor_converter + @direction_key = direction_key + @has_another_page = false + @at_last_page = false + @at_first_page = false + @cursor_attributes = decode_cursor_attributes(cursor) + + set_pagination_helper_flags! + end + + # rubocop: disable CodeReuse/ActiveRecord + def records + @records ||= begin + items = if paginate_backward? + reversed_order + .apply_cursor_conditions(keyset_scope, cursor_attributes) + .reorder(reversed_order) + .limit(per_page_plus_one) + .to_a + else + order + .apply_cursor_conditions(keyset_scope, cursor_attributes) + .limit(per_page_plus_one) + .to_a + end + + @has_another_page = items.size == per_page_plus_one + items.pop if @has_another_page + items.reverse! if paginate_backward? + items + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # This and has_previous_page? methods are direction aware. In case we paginate backwards, + # has_next_page? will mean that we have a previous page. + def has_next_page? + records + + if at_last_page? + false + elsif paginate_forward? + @has_another_page + elsif paginate_backward? + true + end + end + + def has_previous_page? + records + + if at_first_page? + false + elsif paginate_backward? + @has_another_page + elsif paginate_forward? + true + end + end + + def cursor_for_next_page + if has_next_page? + data = order.cursor_attributes_for_node(records.last) + data[direction_key] = FORWARD_DIRECTION + cursor_converter.dump(data) + else + nil + end + end + + def cursor_for_previous_page + if has_previous_page? + data = order.cursor_attributes_for_node(records.first) + data[direction_key] = BACKWARD_DIRECTION + cursor_converter.dump(data) + end + end + + def cursor_for_first_page + cursor_converter.dump({ direction_key => FORWARD_DIRECTION }) + end + + def cursor_for_last_page + cursor_converter.dump({ direction_key => BACKWARD_DIRECTION }) + end + + delegate :each, :empty?, :any?, to: :records + + private + + attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes + + delegate :reversed_order, to: :order + + def at_last_page? + @at_last_page + end + + def at_first_page? + @at_first_page + end + + def per_page_plus_one + per_page + 1 + end + + def decode_cursor_attributes(cursor) + cursor.blank? ? {} : cursor_converter.parse(cursor) + end + + def set_pagination_helper_flags! + @direction = cursor_attributes.delete(direction_key.to_s) + + if cursor_attributes.blank? && @direction.blank? + @at_first_page = true + @direction = FORWARD_DIRECTION + elsif cursor_attributes.blank? + if paginate_forward? + @at_first_page = true + else + @at_last_page = true + end + end + end + + def paginate_backward? + @direction == BACKWARD_DIRECTION + end + + def paginate_forward? + @direction == FORWARD_DIRECTION + end + + def build_scope(scope) + keyset_aware_scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) + + raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + + keyset_aware_scope + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 5ac5737c3be..76d6bbadaa4 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -26,6 +26,8 @@ module Gitlab def build order = if order_values.empty? primary_key_descending_order + elsif Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) elsif ordered_by_primary_key? primary_key_order elsif ordered_by_other_column? diff --git a/lib/gitlab/patch/action_dispatch_journey_formatter.rb b/lib/gitlab/patch/action_dispatch_journey_formatter.rb deleted file mode 100644 index 2d3b7bb9923..00000000000 --- a/lib/gitlab/patch/action_dispatch_journey_formatter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Patch - module ActionDispatchJourneyFormatter - def self.prepended(mod) - mod.alias_method(:old_missing_keys, :missing_keys) - mod.remove_method(:missing_keys) - end - - private - - def missing_keys(route, parts) - missing_keys = nil - tests = route.path.requirements_for_missing_keys_check - route.required_parts.each do |key| - case tests[key] - when nil - unless parts[key] - missing_keys ||= [] - missing_keys << key - end - else - unless tests[key].match?(parts[key]) - missing_keys ||= [] - missing_keys << key - end - end - end - missing_keys - end - end - end -end diff --git a/lib/gitlab/patch/global_id.rb b/lib/gitlab/patch/global_id.rb new file mode 100644 index 00000000000..e99f36c7dca --- /dev/null +++ b/lib/gitlab/patch/global_id.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# To support GlobalID arguments that present a model with its old "deprecated" name +# we alter GlobalID so it will correctly find the record with its new model name. +module Gitlab + module Patch + module GlobalID + def initialize(gid, options = {}) + super + + if deprecation = Gitlab::GlobalId::Deprecations.deprecation_for(model_name) + @new_model_name = deprecation.new_model_name + end + end + + def model_name + new_model_name || super + end + + private + + attr_reader :new_model_name + end + end +end diff --git a/lib/gitlab/patch/hangouts_chat_http_override.rb b/lib/gitlab/patch/hangouts_chat_http_override.rb new file mode 100644 index 00000000000..20dc678e251 --- /dev/null +++ b/lib/gitlab/patch/hangouts_chat_http_override.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module HangoutsChatHTTPOverride + attr_reader :uri + + # See https://github.com/enzinia/hangouts-chat/blob/6a509f61a56e757f8f417578b393b94423831ff7/lib/hangouts_chat/http.rb + def post(payload) + httparty_response = Gitlab::HTTP.post( + uri, + body: payload.to_json, + headers: { 'Content-Type' => 'application/json' }, + parse: nil # Disables automatic response parsing + ) + httparty_response.response + # The rest of the integration expects a Net::HTTP response + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 8618d2da77c..16a6c470213 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -21,13 +21,11 @@ module Gitlab 500.html 502.html 503.html - abuse_reports admin api apple-touch-icon-precomposed.png apple-touch-icon.png assets - autocomplete dashboard deploy.html explore @@ -38,7 +36,6 @@ module Gitlab health_check help import - invites jwt login oauth @@ -48,7 +45,6 @@ module Gitlab robots.txt s search - sent_notifications sitemap sitemap.xml sitemap.xml.gz diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 42f43f998c4..5c9b029a107 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -170,7 +170,7 @@ module Gitlab def self.print_by_total_time(result, options = {}) default_options = { sort_method: :total_time, filter_by: :total_time } - RubyProf::FlatPrinter.new(result).print(STDOUT, default_options.merge(options)) + RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6719dc8362b..e52023c4612 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -43,9 +43,20 @@ module Gitlab end end + # rubocop:disable CodeReuse/ActiveRecord def users - super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord + results = super + + if @project.is_a?(Array) + team_members_for_projects = User.joins(:project_authorizations).where(project_authorizations: { project_id: @project }) + results = results.where(id: team_members_for_projects) + else + results = results.where(id: @project.team.members) + end + + results end + # rubocop:enable CodeReuse/ActiveRecord def limited_blobs_count @limited_blobs_count ||= blobs(limit: count_limit).count diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index 45438d9bf7c..a977040ef6f 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,9 +19,6 @@ module Gitlab end def cluster_prometheus_adapter - application = cluster&.application_prometheus - return application if application&.available? - integration = cluster&.integration_prometheus integration if integration&.available? end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index b7d58e05651..b53fdd60606 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -155,6 +155,7 @@ module Gitlab params '<1w 3d 2h 14m>' types Issue, MergeRequest condition do + quick_action_target.supports_time_tracking? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |raw_duration| @@ -177,6 +178,7 @@ module Gitlab params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' types Issue, MergeRequest condition do + quick_action_target.supports_time_tracking? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end parse_params do |raw_time_date| diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index f3c6315cd6a..47c76e98e5c 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -99,7 +99,7 @@ module Gitlab # Allow it to mark as WIP on MR creation page _or_ through MR notes. (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end - command :draft, :wip do + command :draft do @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 8a432edbd78..e62e1172b65 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,12 +11,16 @@ module Gitlab end def cache_key(key) - "#{cache_type}:#{key}:set" + "#{cache_namespace}:#{key}:set" + end + + def new_cache_key(key) + super(key) end def clear_cache!(key) with do |redis| - keys = read(key).map { |value| "#{cache_type}:#{value}" } + keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) redis.pipelined do @@ -24,11 +28,5 @@ module Gitlab end end end - - private - - def cache_type - Gitlab::Redis::Cache::CACHE_NAMESPACE - end end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index a634f12345a..98b66080b42 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,36 +1,16 @@ # frozen_string_literal: true -# please require all dependencies below: -require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? - module Gitlab module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' - DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380' - REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE' - - class << self - def default_url - DEFAULT_REDIS_CACHE_URL - end - - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] - return file_name unless file_name.nil? - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.cache.yml') - return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent - super - end + private - def instrumentation_class - ::Gitlab::Instrumentation::Redis::Cache - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6380' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 42d5167beb3..9e291a73bb6 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,37 +1,21 @@ # frozen_string_literal: true -# please require all dependencies below: +# We need this require for MailRoom require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) +require 'active_support/core_ext/object/blank' module Gitlab module Redis class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab' MAILROOM_NAMESPACE = 'mail_room:gitlab' - DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381' - REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE' - class << self - def default_url - DEFAULT_REDIS_QUEUES_URL - end + private - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] - return file_name if file_name - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.queues.yml') - return file_name if File.file?(file_name) - - # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent - super - end - - def instrumentation_class - ::Gitlab::Instrumentation::Redis::Queues - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6381' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 2848c9f0b59..d62516bd287 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -# please require all dependencies below: -require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) - module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper @@ -10,30 +7,13 @@ module Gitlab USER_SESSIONS_NAMESPACE = 'session:user:gitlab' USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' - DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382' - REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE' - - class << self - def default_url - DEFAULT_REDIS_SHARED_STATE_URL - end - - def config_file_name - # if ENV set for this class, use it even if it points to a file does not exist - file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] - return file_name if file_name - - # otherwise, if config files exists for this class, use it - file_name = config_file_path('redis.shared_state.yml') - return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent - super - end + private - def instrumentation_class - ::Gitlab::Instrumentation::Redis::SharedState - end + def raw_config_hash + config = super + config[:url] = 'redis://localhost:6382' if config[:url].blank? + config end end end diff --git a/lib/gitlab/redis/trace_chunks.rb b/lib/gitlab/redis/trace_chunks.rb new file mode 100644 index 00000000000..a2e77cb5df5 --- /dev/null +++ b/lib/gitlab/redis/trace_chunks.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class TraceChunks < ::Gitlab::Redis::Wrapper + # The data we store on TraceChunks used to be stored on SharedState. + def self.config_fallback + SharedState + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 94ab67ef08a..bbcc2732e89 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true # This file should only be used by sub-classes, not directly by any clients of the sub-classes -# please require all dependencies below: + +# Explicitly load parts of ActiveSupport because MailRoom does not load +# Rails. require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' +require 'active_support/core_ext/string/inflections' module Gitlab module Redis class Wrapper - DEFAULT_REDIS_URL = 'redis://localhost:6379' - REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE' - class << self delegate :params, :url, to: :new @@ -51,33 +51,47 @@ module Gitlab end end - def default_url - DEFAULT_REDIS_URL + def config_file_path(filename) + path = File.join(rails_root, 'config', filename) + return path if File.file?(path) end - # Return the absolute path to a Rails configuration file - # - # We use this instead of `Rails.root` because for certain tasks - # utilizing these classes, `Rails` might not be available. - def config_file_path(filename) - File.expand_path("../../../config/#{filename}", __dir__) + # We need this local implementation of Rails.root because MailRoom + # doesn't load Rails. + def rails_root + File.expand_path('../../..', __dir__) end def config_file_name - # if ENV set for wrapper class, use it even if it points to a file does not exist - file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] - return file_name unless file_name.nil? + [ + # Instance specific config sources: + ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"], + config_file_path("redis.#{store_name.underscore}.yml"), + + # The current Redis instance may have been split off from another one + # (e.g. TraceChunks was split off from SharedState). There are + # installations out there where the lowest priority config source + # (resque.yml) contains bogus values. In those cases, config_file_name + # should resolve to the instance we originated from (the + # "config_fallback") rather than resque.yml. + config_fallback&.config_file_name, + + # Global config sources: + ENV['GITLAB_REDIS_CONFIG_FILE'], + config_file_path('resque.yml') + ].compact.first + end - # otherwise, if config files exists for wrapper class, use it - file_name = config_file_path('resque.yml') - return file_name if File.file?(file_name) + def store_name + name.demodulize + end - # nil will force use of DEFAULT_REDIS_URL when config file is absent + def config_fallback nil end def instrumentation_class - raise NotImplementedError + "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end end @@ -135,7 +149,7 @@ module Gitlab if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: self.class.default_url } + { url: '' } end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ccb4f6e1097..a31f574fad2 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -138,7 +138,8 @@ module Gitlab end def helm_version_regex - @helm_version_regex ||= %r{#{prefixed_semver_regex}}.freeze + # identical to semver_regex, with optional preceding 'v' + @helm_version_regex ||= Regexp.new("\\Av?#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end def unbounded_semver_regex diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index f73ac628bce..a20e9845fe6 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -17,6 +17,11 @@ module Gitlab "#{type}:#{namespace}:set" end + # NOTE Remove as part of #331319 + def new_cache_key(type) + super("#{type}:#{namespace}") + end + def write(key, value) full_key = cache_key(key) diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index b0bcea0ca69..f60cac0aff0 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -15,8 +15,7 @@ module Gitlab :rails_runner, :rake, :sidekiq, - :test_suite, - :unicorn + :test_suite ].freeze class << self @@ -36,11 +35,6 @@ module Gitlab !!defined?(::Puma) end - # For unicorn, we need to check for actual server instances to avoid false positives. - def unicorn? - !!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer)) - end - def sidekiq? !!(defined?(::Sidekiq) && Sidekiq.server?) end @@ -66,7 +60,7 @@ module Gitlab end def web_server? - puma? || unicorn? + puma? end def action_cable? diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb new file mode 100644 index 00000000000..8d9d8415cb1 --- /dev/null +++ b/lib/gitlab/saas.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# This module is used to return various SaaS related configurations +# which may be overridden in other variants of GitLab + +module Gitlab + module Saas + def self.com_url + 'https://gitlab.com' + end + + def self.staging_com_url + 'https://staging.gitlab.com' + end + + def self.subdomain_regex + %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze + end + + def self.dev_url + 'https://dev.gitlab.org' + end + end +end + +Gitlab::Saas.prepend_mod diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 0f2b7b194c9..30cd63e80c0 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -14,15 +14,21 @@ module Gitlab "#{key}:set" end + # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 + def new_cache_key(key) + "#{cache_namespace}:#{key}:set" + end + # Returns the number of keys deleted by Redis def expire(*keys) return 0 if keys.empty? with do |redis| - keys = keys.map { |key| cache_key(key) } + keys_to_expire = keys.map { |key| cache_key(key) } + keys_to_expire += keys.map { |key| new_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + redis.unlink(*keys_to_expire) end end end @@ -73,5 +79,9 @@ module Gitlab def with(&blk) Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end + + def cache_namespace + Gitlab::Redis::Cache::CACHE_NAMESPACE + end end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 3ac20724403..7ed1958a8d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -129,7 +129,7 @@ module Gitlab config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path + config[:bin_dir] = File.join(gitaly_dir, '_build', 'bin') # binaries by default are in `_build/bin` config[:gitlab] = { url: Gitlab.config.gitlab.url } config[:logging] = { dir: Rails.root.join('log').to_s } @@ -153,8 +153,14 @@ module Gitlab second_storage_nodes = [{ storage: 'test_second_storage', address: "unix:#{gitaly_dir}/gitaly2.socket", primary: true, token: 'secret' }] storages = [{ name: 'default', node: nodes }, { name: 'test_second_storage', node: second_storage_nodes }] - failover = { enabled: false } - config = { socket_path: "#{gitaly_dir}/praefect.socket", memory_queue_enabled: true, virtual_storage: storages, failover: failover } + failover = { enabled: false, election_strategy: 'local' } + config = { + i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, + socket_path: "#{gitaly_dir}/praefect.socket", + memory_queue_enabled: true, + virtual_storage: storages, + failover: failover + } config[:token] = 'secret' if Rails.env.test? TomlRB.dump(config) diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 9490d543dd1..e20834fa912 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -22,7 +22,7 @@ module Gitlab CommandError = Class.new(StandardError) - def initialize(log_output = STDERR) + def initialize(log_output = $stderr) require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter' # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency @@ -47,12 +47,6 @@ module Gitlab option_parser.parse!(argv) - # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - if @queue_selector && @experimental_queue_selector - raise CommandError, - 'You cannot specify --queue-selector and --experimental-queue-selector together' - end - worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path) worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) @@ -63,8 +57,7 @@ module Gitlab # as a worker attribute query, and resolve the queues for the # queue group using this query. - # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - if @queue_selector || @experimental_queue_selector + if @queue_selector SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas) else SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues) @@ -194,11 +187,6 @@ module Gitlab @queue_selector = queue_selector end - # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 - opt.on('--experimental-queue-selector', 'DEPRECATED: use --queue-selector-instead') do |experimental_queue_selector| - @experimental_queue_selector = experimental_queue_selector - end - opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do @negate_queues = true end diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb index 946296a24d3..0670e5521df 100644 --- a/lib/gitlab/sidekiq_config/worker_router.rb +++ b/lib/gitlab/sidekiq_config/worker_router.rb @@ -40,7 +40,7 @@ module Gitlab # queue defined in the input routing rules. The input routing rules, as # described above, is an order-matter array of tuples [query, queue_name]. # - # - The query syntax is the same as the "queue selector" detailedly + # - The query syntax follows "worker matching query" detailedly # denoted in doc/administration/operations/extra_sidekiq_processes.md. # # - The queue_name must be a valid Sidekiq queue name. If the queue name diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index 6f8cc1c60e9..cfe91b9a266 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -14,6 +14,9 @@ module Gitlab job = job.except('error_backtrace', 'error_class', 'error_message') job['class'] = job.delete('wrapped') if job['wrapped'].present? + job['job_size_bytes'] = Sidekiq.dump_json(job['args']).bytesize + job['args'] = ['[COMPRESSED]'] if ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compressed?(job) + # Add process id params job['pid'] = ::Process.pid diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 87fb36d04e9..32194c4926e 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -55,8 +55,6 @@ module Gitlab scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s - payload['job_size_bytes'] = Sidekiq.dump_json(job).bytesize - payload end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index c5b980769f0..30741f29563 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -9,6 +9,8 @@ module Gitlab # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true) lambda do |chain| + # Size limiter should be placed at the top + chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server chain.add ::Gitlab::SidekiqMiddleware::Monitor chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger @@ -18,6 +20,7 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger + chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware if load_balancing_enabled? chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware @@ -39,9 +42,13 @@ module Gitlab # Size limiter should be placed at the bottom, but before the metrics midleware chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Client chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics + chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware if load_balancing_enabled? end end + + def self.load_balancing_enabled? + ::Gitlab::Database::LoadBalancing.enable? + end + private_class_method :load_balancing_enabled? end end - -Gitlab::SidekiqMiddleware.singleton_class.prepend_mod_with('Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 79ac853ea0c..4cf540ce3b8 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -19,6 +19,7 @@ module Gitlab class DuplicateJob DUPLICATE_KEY_TTL = 6.hours DEFAULT_STRATEGY = :until_executing + STRATEGY_NONE = :none attr_reader :existing_jid @@ -51,6 +52,8 @@ module Gitlab end end + job['idempotency_key'] = idempotency_key + self.existing_jid = read_jid.value end @@ -100,6 +103,7 @@ module Gitlab def strategy return DEFAULT_STRATEGY unless worker_klass return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?) + return STRATEGY_NONE unless worker_klass.deduplication_enabled? worker_klass.get_deduplicate_strategy end @@ -117,7 +121,7 @@ module Gitlab end def idempotency_key - @idempotency_key ||= "#{namespace}:#{idempotency_hash}" + @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}" end def idempotency_hash @@ -129,6 +133,10 @@ module Gitlab end def idempotency_string + # TODO: dump the argument's JSON using `Sidekiq.dump_json` instead + # this should be done in the next release so all jobs are written + # with their idempotency key. + # see https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1090 "#{worker_class_name}:#{arguments.join('-')}" end end diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index b542aa4fe4c..1f0c63c5fff 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -3,24 +3,6 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger - def self.keys - @keys ||= [ - :cpu_s, - :gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - :elasticsearch_timed_out_count, - *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, - *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS - ] - end - def call(worker, job, queue) ::Gitlab::InstrumentationHelper.init_instrumentation_data @@ -37,7 +19,6 @@ module Gitlab # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 job[:instrumentation] = {}.tap do |instrumentation_values| ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values) - instrumentation_values.slice!(*self.class.keys) end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 474afffcf93..6d130957f36 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -13,6 +13,10 @@ module Gitlab @metrics = init_metrics @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + + if ::Gitlab::Database::LoadBalancing.enable? + @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') + end end def call(worker, job, queue) @@ -69,6 +73,15 @@ module Gitlab @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) + + if ::Gitlab::Database::LoadBalancing.enable? && job[:database_chosen] + load_balancing_labels = { + database_chosen: job[:database_chosen], + data_consistency: job[:data_consistency] + } + + @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) + end end end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb new file mode 100644 index 00000000000..bce295d8ba5 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + class Compressor + PayloadDecompressionConflictError = Class.new(StandardError) + PayloadDecompressionError = Class.new(StandardError) + + # Level 5 is a good trade-off between space and time + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1054#note_568129605 + COMPRESS_LEVEL = 5 + ORIGINAL_SIZE_KEY = 'original_job_size_bytes' + COMPRESSED_KEY = 'compressed' + + def self.compressed?(job) + job&.has_key?(COMPRESSED_KEY) + end + + def self.compress(job, job_args) + compressed_args = Base64.strict_encode64(Zlib::Deflate.deflate(job_args, COMPRESS_LEVEL)) + + job[COMPRESSED_KEY] = true + job[ORIGINAL_SIZE_KEY] = job_args.bytesize + job['args'] = [compressed_args] + + compressed_args + end + + def self.decompress(job) + return unless compressed?(job) + + validate_args!(job) + + job.except!(ORIGINAL_SIZE_KEY, COMPRESSED_KEY) + job['args'] = Sidekiq.load_json(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) + rescue Zlib::Error + raise PayloadDecompressionError, 'Fail to decompress Sidekiq job payload' + end + + def self.validate_args!(job) + if job['args'] && job['args'].length != 1 + exception = PayloadDecompressionConflictError.new('Sidekiq argument list should include 1 argument.\ + This means that there is another a middleware interfering with the job payload.\ + That conflicts with the payload compressor') + ::Gitlab::ErrorTracking.track_and_raise_exception(exception) + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/server.rb b/lib/gitlab/sidekiq_middleware/size_limiter/server.rb new file mode 100644 index 00000000000..70b384c8f28 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/server.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + class Server + def call(worker, job, queue) + # This middleware should always decompress jobs regardless of the + # limiter mode or size limit. Otherwise, this could leave compressed + # payloads in queues that are then not able to be processed. + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.decompress(job) + + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index 2c50c4a2157..d86f1609f14 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -3,76 +3,103 @@ module Gitlab module SidekiqMiddleware module SizeLimiter - # Validate a Sidekiq job payload limit based on current configuration. + # Handle a Sidekiq job payload limit based on current configuration. # This validator pulls the configuration from the environment variables: - # # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size - # limiter. This must be either `track` or `raise`. - # + # limiter. This must be either `track` or `compress`. + # - GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES: the + # threshold before the input job payload is compressed. # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. # - # If the size of job payload after serialization exceeds the limit, an - # error is tracked raised adhering to the mode. + # In track mode, if a job payload limit exceeds the size limit, an + # event is sent to Sentry and the job is scheduled like normal. + # + # In compress mode, if a job payload limit exceeds the threshold, it is + # then compressed. If the compressed payload still exceeds the limit, the + # job is discarded, and a ExceedLimitError exception is raised. class Validator def self.validate!(worker_class, job) new(worker_class, job).validate! end DEFAULT_SIZE_LIMIT = 0 + DEFAULT_COMPRESION_THRESHOLD_BYTES = 100_000 # 100kb MODES = [ TRACK_MODE = 'track', - RAISE_MODE = 'raise' + COMPRESS_MODE = 'compress' ].freeze - attr_reader :mode, :size_limit + attr_reader :mode, :size_limit, :compression_threshold def initialize( worker_class, job, mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], + compression_threshold: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES'], size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] ) @worker_class = worker_class @job = job + set_mode(mode) + set_compression_threshold(compression_threshold) + set_size_limit(size_limit) + end + + def validate! + return unless @size_limit > 0 + return if allow_big_payload? + + job_args = compress_if_necessary(::Sidekiq.dump_json(@job['args'])) + return if job_args.bytesize <= @size_limit + + exception = exceed_limit_error(job_args) + if compress_mode? + raise exception + else + track(exception) + end + end + + private + + def set_mode(mode) @mode = (mode || TRACK_MODE).to_s.strip unless MODES.include?(@mode) ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode." @mode = TRACK_MODE end + end + + def set_compression_threshold(compression_threshold) + @compression_threshold = (compression_threshold || DEFAULT_COMPRESION_THRESHOLD_BYTES).to_i + if @compression_threshold <= 0 + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter compression threshold: #{@compression_threshold}" + @compression_threshold = DEFAULT_COMPRESION_THRESHOLD_BYTES + end + end + def set_size_limit(size_limit) @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i if @size_limit < 0 ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" end end - def validate! - return unless @size_limit > 0 - - return if allow_big_payload? - return if job_size <= @size_limit - - exception = ExceedLimitError.new(@worker_class, job_size, @size_limit) - # This should belong to Gitlab::ErrorTracking. We'll remove this - # after this epic is done: - # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 - exception.set_backtrace(backtrace) - - if raise_mode? - raise exception - else - track(exception) + def exceed_limit_error(job_args) + ExceedLimitError.new(@worker_class, job_args.bytesize, @size_limit).tap do |exception| + # This should belong to Gitlab::ErrorTracking. We'll remove this + # after this epic is done: + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 + exception.set_backtrace(backtrace) end end - private + def compress_if_necessary(job_args) + return job_args unless compress_mode? + return job_args if job_args.bytesize < @compression_threshold - def job_size - # This maynot be the optimal solution, but can be acceptable solution - # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere. - # There is no clean way to intefere to prevent double serialization. - @job_size ||= ::Sidekiq.dump_json(@job).bytesize + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compress(@job, job_args) end def allow_big_payload? @@ -80,8 +107,8 @@ module Gitlab worker_class.respond_to?(:big_payload?) && worker_class.big_payload? end - def raise_mode? - @mode == RAISE_MODE + def compress_mode? + @mode == COMPRESS_MODE end def track(exception) diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index b8affb42372..d28b5fb509a 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -63,7 +63,7 @@ module Gitlab # Convert Markdown to slacks format def format(string) - Slack::Messenger::Util::LinkFormatter.format(string) + ::Slack::Messenger::Util::LinkFormatter.format(string) end def resource_url diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb index 4b7d93c91ce..97f52491e9e 100644 --- a/lib/gitlab/stack_prof.rb +++ b/lib/gitlab/stack_prof.rb @@ -118,7 +118,6 @@ module Gitlab # # see also: # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals - # * https://github.com/phusion/unicorn/blob/master/SIGNALS # * https://github.com/mperham/sidekiq/wiki/Signals Signal.trap('SIGUSR2') do write.write('.') diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 1ceccc64ec0..227962fc0f7 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -61,7 +61,7 @@ module Gitlab def prompt(message, choices = nil) begin print(message) - answer = STDIN.gets.chomp + answer = $stdin.gets.chomp end while choices.present? && !choices.include?(answer) answer end @@ -70,12 +70,12 @@ module Gitlab # # message - custom message to display before input def prompt_for_password(message = 'Enter password: ') - unless STDIN.tty? + unless $stdin.tty? print(message) - return STDIN.gets.chomp + return $stdin.gets.chomp end - STDIN.getpass(message) + $stdin.getpass(message) end # Runs the given command and matches the output against the given pattern diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e1ca4b5ff6a..e302865c897 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -5,11 +5,19 @@ module Gitlab class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze + TEMPLATES_WITH_LATEST_VERSION = { + 'Jobs/Browser-Performance-Testing' => true, + 'Security/API-Fuzzing' => true, + 'Security/DAST' => true, + 'Terraform' => true + }.freeze + def description "# This file is a template, and might need editing before it works on your project." end class << self + extend ::Gitlab::Utils::Override include Gitlab::Utils::StrongMemoize def extension @@ -54,6 +62,31 @@ module Gitlab excluded_patterns: self.excluded_patterns ) end + + override :find + def find(key, project = nil) + if try_redirect_to_latest?(key, project) + key += '.latest' + end + + super(key, project) + end + + private + + # To gauge the impact of the latest template, + # you can redirect the stable template to the latest template by enabling the feature flag. + # See https://docs.gitlab.com/ee/development/cicd/templates.html#versioning for more information. + def try_redirect_to_latest?(key, project) + return false unless templates_with_latest_version[key] + + flag_name = "redirect_to_latest_template_#{key.underscore.tr('/', '_')}" + ::Feature.enabled?(flag_name, project, default_enabled: :yaml) + end + + def templates_with_latest_version + TEMPLATES_WITH_LATEST_VERSION + end end end end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 16e7b8a7eca..ac1522b8a6c 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,21 +10,21 @@ module Gitlab APPLICATION_DEFAULT = 1 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class, :css_filename) + Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'), - Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'), - Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'), - Theme.new(5, 'Green', 'ui-green', 'theme_green'), - Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'), - Theme.new(9, 'Red', 'ui-red', 'theme_red'), - Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'), - Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'), - Theme.new(3, 'Light', 'ui-light', 'theme_light'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil) + Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo', '#292961'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), + Theme.new(4, 'Blue', 'ui-blue', 'theme_blue', '#1a3652'), + Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue', '#2261a1'), + Theme.new(5, 'Green', 'ui-green', 'theme_green', '#0d4524'), + Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green', '#156b39'), + Theme.new(9, 'Red', 'ui-red', 'theme_red', '#691a16'), + Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red', '#a62e21'), + Theme.new(2, 'Dark', 'ui-dark', 'theme_dark', '#303030'), + Theme.new(3, 'Light', 'ui-light', 'theme_light', '#666'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil, '#303030') ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index bfdfb01093f..67ecf498cf7 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -24,6 +24,12 @@ module Gitlab end def output(seconds) + seconds.to_i < 0 ? negative_output(seconds) : positive_output(seconds) + end + + private + + def positive_output(seconds) ChronicDuration.output( seconds, CUSTOM_DAY_AND_MONTH_LENGTH.merge( @@ -34,7 +40,9 @@ module Gitlab nil end - private + def negative_output(seconds) + "-" + positive_output(seconds.abs) + end def limit_to_hours_setting Gitlab::CurrentSettings.time_tracking_limit_to_hours diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 4c40bfbc06f..3ec06fba5d1 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -22,9 +22,7 @@ module Gitlab }.freeze class Aggregate - delegate :weekly_time_range, - :monthly_time_range, - to: Gitlab::UsageDataCounters::HLLRedisCounter + include Gitlab::Usage::TimeFrame def initialize(recorded_at) @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) @@ -32,15 +30,15 @@ module Gitlab end def all_time_data - aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) + aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) end def monthly_data - aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) end def weekly_data - aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME)) end private @@ -54,7 +52,7 @@ module Gitlab case aggregation[:source] when REDIS_SOURCE - if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME + if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK Gitlab::ErrorTracking .track_and_raise_for_dev_exception( @@ -64,8 +62,6 @@ module Gitlab data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) end when DATABASE_SOURCE - next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) else Gitlab::ErrorTracking diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index 3069afab147..eccf79b9703 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -56,15 +56,15 @@ module Gitlab end def time_period_to_human_name(time_period) - return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank? + return Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME if time_period.blank? start_date = time_period.first.to_date end_date = time_period.last.to_date if (end_date - start_date).to_i > 7 - Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME + Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME else - Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME + Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 29b44f2bd0a..7b5bee3f8bd 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -6,11 +6,14 @@ module Gitlab module Instrumentations class BaseMetric include Gitlab::Utils::UsageData + include Gitlab::Usage::TimeFrame attr_reader :time_frame + attr_reader :options - def initialize(time_frame:) + def initialize(time_frame:, options: {}) @time_frame = time_frame + @options = options end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index f83f90dea03..69a288e5b6e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -43,16 +43,28 @@ module Gitlab finish: self.class.metric_finish&.call) end - def relation - self.class.metric_relation.call.where(time_constraints) + def to_sql + Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column) + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation, + relation: relation, + column: self.class.column + ) end private + def relation + self.class.metric_relation.call.where(time_constraints) + end + def time_constraints case time_frame when '28d' - { created_at: 30.days.ago..2.days.ago } + monthly_time_range_db_params when 'all' {} when 'none' diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 7c97cc37d17..1849773e33d 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -13,6 +13,9 @@ module Gitlab # end # end class << self + attr_reader :metric_operation + @metric_operation = :alt + def value(&block) @metric_value = block end @@ -25,6 +28,12 @@ module Gitlab self.class.metric_value.call end end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index 140d56f0d42..a36e612a1cb 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -7,35 +7,50 @@ module Gitlab class RedisHLLMetric < BaseMetric # Usage example # - # class CountUsersVisitingAnalyticsValuestreamMetric < RedisHLLMetric - # event_names :g_analytics_valuestream + # In metric YAML defintion + # instrumentation_class: RedisHLLMetric + # events: + # - g_analytics_valuestream # end class << self - def event_names(events = nil) - @metric_events = events - end + attr_reader :metric_operation + @metric_operation = :redis + end - attr_reader :metric_events + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "options events are required" unless metric_events.present? + end + + def metric_events + options[:events] end def value redis_usage_data do - event_params = time_constraints.merge(event_names: self.class.metric_events) + event_params = time_constraints.merge(event_names: metric_events) Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(**event_params) end end + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end + private def time_constraints case time_frame when '28d' - { start_date: 4.weeks.ago.to_date, end_date: Date.current } + monthly_time_range when '7d' - { start_date: 7.days.ago.to_date, end_date: Date.current } + weekly_time_range else - raise "Unknown time frame: #{time_frame} for TimeConstraint" + raise "Unknown time frame: #{time_frame} for RedisHLLMetric" end end end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb new file mode 100644 index 00000000000..0728af9e2ca --- /dev/null +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class NameSuggestion + FREE_TEXT_METRIC_NAME = "<please fill metric name>" + REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" + CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" + + class << self + def for(operation, relation: nil, column: nil) + case operation + when :count + name_suggestion(column: column, relation: relation, prefix: 'count') + when :distinct_count + name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + when :estimate_batch_distinct_count + name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + when :sum + name_suggestion(column: column, relation: relation, prefix: 'sum') + when :redis + REDIS_EVENT_METRIC_NAME + when :alt + FREE_TEXT_METRIC_NAME + else + raise ArgumentError, "#{operation} operation not supported" + end + end + + private + + def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) + # rubocop: disable CodeReuse/ActiveRecord + relation = relation.unscope(where: :created_at) + # rubocop: enable CodeReuse/ActiveRecord + + parts = [prefix] + arel_column = arelize_column(relation, column) + + # nil as column indicates that the counting would use fallback value of primary key. + # Because counting primary key from relation is the conceptual equal to counting all + # records from given relation, in order to keep name suggestion more condensed + # primary key column is skipped. + # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not + # as count_id_from_issues since it does not add more information to the name suggestion + if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] + parts << arel_column.name + parts << 'from' + end + + arel = arel_query(relation: relation, column: arel_column, distinct: distinct) + constraints = parse_constraints(relation: relation, arel: arel) + + # In some cases due to performance reasons metrics are instrumented with joined relations + # where relation listed in FROM statement is not the one that includes counted attribute + # in such situations to make name suggestion more intuitive source should be inferred based + # on the relation that provide counted attribute + # EG: SELECT COUNT(deployments.environment_id) FROM clusters + # JOIN deployments ON deployments.cluster_id = cluster.id + # should be translated into: + # count_environment_id_from_deployments_with_clusters + # instead of + # count_environment_id_from_clusters_with_deployments + actual_source = parse_source(relation, arel_column) + + append_constraints_prompt(actual_source, [constraints], parts) + + parts << actual_source + parts += process_joined_relations(actual_source, arel, relation, constraints) + parts.compact.join('_').delete('"') + end + + def append_constraints_prompt(target, constraints, parts) + applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + return unless applicable_constraints.any? + + parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } + end + + def parse_constraints(relation:, arel:) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + .new(connection) + .accept(arel, collector(connection)) + .value + end + + # TODO: joins with `USING` keyword + def process_joined_relations(actual_source, arel, relation, where_constraints) + joins = parse_joins(connection: relation.connection, arel: arel) + return [] unless joins.any? + + sources = [relation.table_name, *joins.map { |join| join[:source] }] + joins = extract_joins_targets(joins, sources) + + relations = if actual_source != relation.table_name + build_relations_tree(joins + [{ source: relation.table_name }], actual_source) + else + # in case where counter attribute comes from joined relations, the relations + # diagram has to be built bottom up, thus source and target are reverted + build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) + end + + collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) + end + + def parse_joins(connection:, arel:) + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins + .new(connection) + .accept(arel) + end + + def extract_joins_targets(joins, sources) + joins.map do |join| + source_regex = /(#{join[:source]})\.(\w+_)*id/i + + tables_except_src = (sources - [join[:source]]).join('|') + target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i + + join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i + matched = join_cond_regex.match(join[:constraints]) + + if matched + join[:target] = matched[:target] + join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') + end + + join + end + end + + def build_relations_tree(joins, parent, source_key: :source, target_key: :target) + return [] if joins.blank? + + tree = {} + tree[parent] = [] + + joins.each do |join| + if join[source_key] == parent + tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) + end + end + tree + end + + def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) + conjunction = conjunctions.next + relations.each do |subtree| + subtree.each do |parent, children| + parts << "<#{conjunction}>" + join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) + append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + parts << parent + collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) + end + end + parts + end + + def arelize_column(relation, column) + case column + when Arel::Attribute + column + when NilClass + Arel::Table.new(relation.table_name)[relation.primary_key] + when String + if column.include?('.') + table, col = column.split('.') + Arel::Table.new(table)[col] + else + Arel::Table.new(relation.table_name)[column] + end + when Symbol + arelize_column(relation, column.to_s) + end + end + + def parse_source(relation, column) + column.relation.name || relation.table_name + end + + def collector(connection) + Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) + end + + def arel_query(relation:, column: nil, distinct: nil) + column ||= relation.primary_key + + if column.is_a?(Arel::Attribute) + relation.select(column.count(distinct)).arel + else + relation.select(relation.all.table[column].count(distinct)).arel + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 49581169452..a669b43f395 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -5,10 +5,6 @@ module Gitlab module Metrics module NamesSuggestions class Generator < ::Gitlab::UsageData - FREE_TEXT_METRIC_NAME = "<please fill metric name>" - REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" - CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" - class << self def generate(key_path) uncached_data.deep_stringify_keys.dig(*key_path.split('.')) @@ -17,200 +13,36 @@ module Gitlab private def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count') + Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + Gitlab::Usage::Metrics::NameSuggestion.for(:distinct_count, column: column, relation: relation) end def redis_usage_counter - REDIS_EVENT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end def alt_usage_data(*) - FREE_TEXT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) end def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME } + counter.fallback_totals.transform_values { |_| Gitlab::Usage::Metrics::NameSuggestion.for(:redis) } end def sum(relation, column, *rest) - name_suggestion(column: column, relation: relation, prefix: 'sum') + Gitlab::Usage::Metrics::NameSuggestion.for(:sum, column: column, relation: relation) end def estimate_batch_distinct_count(relation, column = nil, *rest) - name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + Gitlab::Usage::Metrics::NameSuggestion.for(:estimate_batch_distinct_count, column: column, relation: relation) end def add(*args) "add_#{args.join('_and_')}" end - - def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - # rubocop: disable CodeReuse/ActiveRecord - relation = relation.unscope(where: :created_at) - # rubocop: enable CodeReuse/ActiveRecord - - parts = [prefix] - arel_column = arelize_column(relation, column) - - # nil as column indicates that the counting would use fallback value of primary key. - # Because counting primary key from relation is the conceptual equal to counting all - # records from given relation, in order to keep name suggestion more condensed - # primary key column is skipped. - # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not - # as count_id_from_issues since it does not add more information to the name suggestion - if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] - parts << arel_column.name - parts << 'from' - end - - arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - constraints = parse_constraints(relation: relation, arel: arel) - - # In some cases due to performance reasons metrics are instrumented with joined relations - # where relation listed in FROM statement is not the one that includes counted attribute - # in such situations to make name suggestion more intuitive source should be inferred based - # on the relation that provide counted attribute - # EG: SELECT COUNT(deployments.environment_id) FROM clusters - # JOIN deployments ON deployments.cluster_id = cluster.id - # should be translated into: - # count_environment_id_from_deployments_with_clusters - # instead of - # count_environment_id_from_clusters_with_deployments - actual_source = parse_source(relation, arel_column) - - append_constraints_prompt(actual_source, [constraints], parts) - - parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, constraints) - parts.compact.join('_').delete('"') - end - - def append_constraints_prompt(target, constraints, parts) - applicable_constraints = constraints.select { |constraint| constraint.include?(target) } - return unless applicable_constraints.any? - - parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } - end - - def parse_constraints(relation:, arel:) - connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints - .new(connection) - .accept(arel, collector(connection)) - .value - end - - # TODO: joins with `USING` keyword - def process_joined_relations(actual_source, arel, relation, where_constraints) - joins = parse_joins(connection: relation.connection, arel: arel) - return [] unless joins.any? - - sources = [relation.table_name, *joins.map { |join| join[:source] }] - joins = extract_joins_targets(joins, sources) - - relations = if actual_source != relation.table_name - build_relations_tree(joins + [{ source: relation.table_name }], actual_source) - else - # in case where counter attribute comes from joined relations, the relations - # diagram has to be built bottom up, thus source and target are reverted - build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) - end - - collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) - end - - def parse_joins(connection:, arel:) - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins - .new(connection) - .accept(arel) - end - - def extract_joins_targets(joins, sources) - joins.map do |join| - source_regex = /(#{join[:source]})\.(\w+_)*id/i - - tables_except_src = (sources - [join[:source]]).join('|') - target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i - - join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i - matched = join_cond_regex.match(join[:constraints]) - - if matched - join[:target] = matched[:target] - join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') - end - - join - end - end - - def build_relations_tree(joins, parent, source_key: :source, target_key: :target) - return [] if joins.blank? - - tree = {} - tree[parent] = [] - - joins.each do |join| - if join[source_key] == parent - tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) - end - end - tree - end - - def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) - conjunction = conjunctions.next - relations.each do |subtree| - subtree.each do |parent, children| - parts << "<#{conjunction}>" - join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) - parts << parent - collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) - end - end - parts - end - - def arelize_column(relation, column) - case column - when Arel::Attribute - column - when NilClass - Arel::Table.new(relation.table_name)[relation.primary_key] - when String - if column.include?('.') - table, col = column.split('.') - Arel::Table.new(table)[col] - else - Arel::Table.new(relation.table_name)[column] - end - when Symbol - arelize_column(relation, column.to_s) - end - end - - def parse_source(relation, column) - column.relation.name || relation.table_name - end - - def collector(connection) - Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) - end - - def arel_query(relation:, column: nil, distinct: nil) - column ||= relation.primary_key - - if column.is_a?(Arel::Attribute) - relation.select(column.count(distinct)).arel - else - relation.select(relation.all.table[column].count(distinct)).arel - end - end end end end diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb new file mode 100644 index 00000000000..f6947c4c8ff --- /dev/null +++ b/lib/gitlab/usage/metrics/query.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class Query + class << self + def for(operation, relation, column = nil, **extra) + case operation + when :count + count(relation, column) + when :distinct_count + distinct_count(relation, column) + when :sum + sum(relation, column) + when :estimate_batch_distinct_count + estimate_batch_distinct_count(relation, column) + when :histogram + histogram(relation, column, **extra) + else + raise ArgumentError, "#{operation} operation not supported" + end + end + + private + + def count(relation, column = nil) + raw_sql(relation, column) + end + + def distinct_count(relation, column = nil) + raw_sql(relation, column, true) + end + + def sum(relation, column) + relation.select(relation.all.table[column].sum).to_sql + end + + def estimate_batch_distinct_count(relation, column = nil) + raw_sql(relation, column, true) + end + + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + bucket_segments = bucket_size - 1 + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + query.to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def raw_sql(relation, column, distinct = false) + column ||= relation.primary_key + relation.select(relation.all.table[column].count(distinct)).to_sql + end + end + end + end + end +end diff --git a/lib/gitlab/usage/time_frame.rb b/lib/gitlab/usage/time_frame.rb new file mode 100644 index 00000000000..966a087ee07 --- /dev/null +++ b/lib/gitlab/usage/time_frame.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module TimeFrame + ALL_TIME_TIME_FRAME_NAME = "all" + SEVEN_DAYS_TIME_FRAME_NAME = "7d" + TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" + + def weekly_time_range + { start_date: 7.days.ago.to_date, end_date: Date.current } + end + + def monthly_time_range + { start_date: 4.weeks.ago.to_date, end_date: Date.current } + end + + # This time range is skewed for batch counter performance. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972 + def monthly_time_range_db_params(column: :created_at) + { column => 30.days.ago..2.days.ago } + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b1ba529d4a4..415a5bff261 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -33,6 +33,7 @@ module Gitlab class << self include Gitlab::Utils::UsageData include Gitlab::Utils::StrongMemoize + include Gitlab::Usage::TimeFrame def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do @@ -55,7 +56,7 @@ module Gitlab .merge(object_store_usage_data) .merge(topology_usage_data) .merge(usage_activity_by_stage) - .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) + .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) .merge(analytics_unique_visits_data) .merge(compliance_unique_visits_data) .merge(search_unique_visits_data) @@ -165,7 +166,6 @@ module Gitlab projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), - projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), @@ -188,7 +188,6 @@ module Gitlab services_usage, usage_counters, user_preferences_usage, - ingress_modsecurity_usage, container_expiration_policies_usage, service_desk_counts, email_campaign_counts @@ -228,16 +227,17 @@ module Gitlab { counts_monthly: { # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment.where(last_28_days_time_period)), - successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), - failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), + deployments: deployment_count(Deployment.where(monthly_time_range_db_params)), + successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)), + failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)), # rubocop: enable UsageData/LargeTable: - packages: count(::Packages::Package.where(last_28_days_time_period)), - personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), - project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) + projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)), + packages: count(::Packages::Package.where(monthly_time_range_db_params)), + personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), + project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) + snowplow_event_counts(monthly_time_range_db_params(column: :collector_tstamp)) ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -294,7 +294,6 @@ module Gitlab reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, - ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } @@ -376,29 +375,6 @@ module Gitlab Gitlab::UsageData::Topology.new.topology_usage_data end - # rubocop: disable UsageData/DistinctCountByLargeForeignKey - def ingress_modsecurity_usage - ## - # This method measures usage of the Modsecurity Web Application Firewall across the entire - # instance's deployed environments. - # - # NOTE: this service is an approximation as it does not yet take into account if environment - # is enabled and only measures applications installed using GitLab Managed Apps (disregards - # CI-based managed apps). - # - # More details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28331#note_318621786 - ## - - column = ::Deployment.arel_table[:environment_id] - { - ingress_modsecurity_logging: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.logging), column), - ingress_modsecurity_blocking: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.blocking), column), - ingress_modsecurity_disabled: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_disabled), column), - ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column) - } - end - # rubocop: enable UsageData/DistinctCountByLargeForeignKey - # rubocop: disable CodeReuse/ActiveRecord def container_expiration_policies_usage results = {} @@ -427,15 +403,15 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Integration.available_services_names(include_dev: false).each_with_object({}) do |service_name, response| - service_type = Integration.service_name_to_type(service_name) - - response["projects_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where(type: service_type)) - response["groups_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where(type: service_type)) - response["templates_#{service_name}_active".to_sym] = count(Integration.active.where(template: true, type: service_type)) - response["instances_#{service_name}_active".to_sym] = count(Integration.active.where(instance: true, type: service_type)) - response["projects_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: service_type)) - response["groups_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: service_type)) + Integration.available_services_names(include_dev: false).each_with_object({}) do |name, response| + type = Integration.integration_name_to_type(name) + + response[:"projects_#{name}_active"] = count(Integration.active.where.not(project: nil).where(type: type)) + response[:"groups_#{name}_active"] = count(Integration.active.where.not(group: nil).where(type: type)) + response[:"templates_#{name}_active"] = count(Integration.active.where(template: true, type: type)) + response[:"instances_#{name}_active"] = count(Integration.active.where(instance: true, type: type)) + response[:"projects_inheriting_#{name}_active"] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: type)) + response[:"groups_inheriting_#{name}_active"] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: type)) end.merge(jira_usage, jira_import_usage) # rubocop: enable UsageData/LargeTable: end @@ -521,10 +497,6 @@ module Gitlab "#{platform}-#{ohai_data['platform_version']}" end - def last_28_days_time_period(column: :created_at) - { column => 30.days.ago..2.days.ago } - end - # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) { @@ -742,7 +714,7 @@ module Gitlab hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } - results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, **monthly_time_range) } { analytics_unique_visits: results } end @@ -752,7 +724,7 @@ module Gitlab hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } - results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, **monthly_time_range) } { compliance_unique_visits: results } end @@ -760,11 +732,11 @@ module Gitlab def search_unique_visits_data events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search') results = events.each_with_object({}) do |event, hash| - hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) } + hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, **weekly_time_range) } end - results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) } - results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **weekly_time_range) } + results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **monthly_time_range) } { search_unique_visits: results } end @@ -852,17 +824,16 @@ module Gitlab sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - series_amount = Namespaces::InProductMarketingEmailsService::INTERVAL_DAYS.count - Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| # rubocop: enable UsageData/LargeTable: + series_amount = Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count 0.upto(series_amount - 1).map do |series| # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails clicked_count = clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count - result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count + result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count unless track == 'experience' end end end @@ -917,7 +888,7 @@ module Gitlab end def project_imports(time_period) - { + counters = { gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), github: projects_imported_count('github', time_period), @@ -928,6 +899,10 @@ module Gitlab manifest: projects_imported_count('manifest', time_period), gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord } + + counters[:total] = add(*counters.values) + + counters end def projects_imported_count(from, time_period) diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index dd66a40a48f..c72f487a442 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -21,6 +21,7 @@ - i_package_golang_delete_package - i_package_golang_pull_package - i_package_golang_push_package +- i_package_helm_pull_package - i_package_maven_delete_package - i_package_maven_pull_package - i_package_maven_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 833eebd5d04..2a231f8fce0 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -38,6 +38,7 @@ module Gitlab # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self include Gitlab::Utils::UsageData + include Gitlab::Usage::TimeFrame # Track unique events # @@ -98,14 +99,6 @@ module Gitlab end end - def weekly_time_range - { start_date: 7.days.ago.to_date, end_date: Date.current } - end - - def monthly_time_range - { start_date: 4.weeks.ago.to_date, end_date: Date.current } - end - def known_event?(event_name) event_for(event_name).present? end diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index cc89fbd5caf..5023161a9dd 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -164,6 +164,11 @@ category: code_review aggregation: weekly # Diff settings events +- name: i_code_review_click_diff_view_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data - name: i_code_review_click_single_file_mode_setting redis_slot: code_review category: code_review @@ -219,3 +224,11 @@ category: code_review aggregation: weekly feature_flag: diff_settings_usage_data +- name: i_code_review_user_load_conflict_ui + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_resolve_conflict + redis_slot: code_review + category: code_review + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index f2504396cc4..f2e45a52434 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -180,7 +180,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_group_code_coverage_project_click_total - name: i_testing_load_performance_widget_total category: testing redis_slot: testing @@ -345,18 +344,15 @@ category: terraform redis_slot: terraform aggregation: weekly - feature_flag: usage_data_p_terraform_state_api_unique_users # Pipeline Authoring - name: o_pipeline_authoring_unique_users_committing_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly - feature_flag: usage_data_unique_users_committing_ciconfigfile - name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly - feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile # Merge request widgets - name: users_expanding_secure_security_report redis_slot: secure diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index adc5ba36ad7..f594c6a1b7c 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -4,22 +4,18 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_cross_reference category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_list_issues category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_create_issue category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_slack_service_issue_notification category: ecosystem redis_slot: ecosystem diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index d1864cd569b..62b0d6dea86 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -182,3 +182,9 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_users_epic_issue_added_from_epic + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index d8ad2b538d6..e5031599dd0 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -47,6 +47,14 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_helm_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_helm_user + category: user_packages + aggregation: weekly + redis_slot: package - name: i_package_maven_deploy_token category: deploy_token_packages aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index eb28a387a97..0d6f4b93aee 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -44,6 +44,8 @@ module Gitlab MR_INCLUDING_CI_CONFIG_ACTION = 'o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile' MR_MILESTONE_CHANGED_ACTION = 'i_code_review_user_milestone_changed' MR_LABELS_CHANGED_ACTION = 'i_code_review_user_labels_changed' + MR_LOAD_CONFLICT_UI_ACTION = 'i_code_review_user_load_conflict_ui' + MR_RESOLVE_CONFLICT_ACTION = 'i_code_review_user_resolve_conflict' class << self def track_mr_diffs_action(merge_request:) @@ -187,7 +189,6 @@ module Gitlab end def track_mr_including_ci_config(user:, merge_request:) - return unless Feature.enabled?(:usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile, user, default_enabled: :yaml) return unless merge_request.includes_ci_config? track_unique_action_by_user(MR_INCLUDING_CI_CONFIG_ACTION, user) @@ -201,6 +202,14 @@ module Gitlab track_unique_action_by_user(MR_LABELS_CHANGED_ACTION, user) end + def track_loading_conflict_ui_action(user:) + track_unique_action_by_user(MR_LOAD_CONFLICT_UI_ACTION, user) + end + + def track_resolve_conflict_action(user:) + track_unique_action_by_user(MR_RESOLVE_CONFLICT_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb index e181da01229..dde5dde19e0 100644 --- a/lib/gitlab/usage_data_metrics.rb +++ b/lib/gitlab/usage_data_metrics.rb @@ -7,9 +7,12 @@ module Gitlab def uncached_data ::Gitlab::Usage::MetricDefinition.all.map do |definition| instrumentation_class = definition.attributes[:instrumentation_class] + options = definition.attributes[:options] if instrumentation_class.present? - metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new(time_frame: definition.attributes[:time_frame]).value + metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new( + time_frame: definition.attributes[:time_frame], + options: options).value metric_payload(definition.key_path, metric_value) else diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 1c776501fdb..da01b68e8fc 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -6,43 +6,20 @@ module Gitlab class UsageDataQueries < UsageData class << self def count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column) + Gitlab::Usage::Metrics::Query.for(:count, relation, column) end def distinct_count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column, :distinct) - end - - def redis_usage_data(counter = nil, &block) - if block_given? - { redis_usage_data_block: block.to_s } - elsif counter.present? - { redis_usage_data_counter: counter } - end + Gitlab::Usage::Metrics::Query.for(:distinct_count, relation, column) end def sum(relation, column, *args, **kwargs) - relation.select(relation.all.table[column].sum).to_sql + Gitlab::Usage::Metrics::Query.for(:sum, relation, column) end # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) - cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - - bucket_segments = bucket_size - 1 - width_bucket = Arel::Nodes::NamedFunction - .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) - .as('buckets') - - query = cte - .table - .project(width_bucket, cte.table[:count]) - .group('buckets') - .order('buckets') - .with(cte.to_arel) - - query.to_sql + Gitlab::Usage::Metrics::Query.for(:histogram, relation, column, buckets: buckets, bucket_size: bucket_size) end # rubocop: enable CodeReuse/ActiveRecord @@ -50,11 +27,11 @@ module Gitlab # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs) - raw_sql(relation, column, :distinct) + Gitlab::Usage::Metrics::Query.for(:estimate_batch_distinct_count, relation, column) end def add(*args) - 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') + 'SELECT ' + args.map { |arg| "(#{arg})" }.join(' + ') end def maximum_id(model, column = nil) @@ -63,6 +40,14 @@ module Gitlab def minimum_id(model, column = nil) end + def redis_usage_data(counter = nil, &block) + if block_given? + { redis_usage_data_block: block.to_s } + elsif counter.present? + { redis_usage_data_counter: counter } + end + end + def jira_service_data { projects_jira_server_active: 0, @@ -73,13 +58,6 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end - - private - - def raw_sql(relation, column, distinct = nil) - column ||= relation.primary_key - relation.select(relation.all.table[column].count(distinct)).to_sql - end end end end diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index ffd12c1b518..dc43d977a62 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -9,7 +9,7 @@ module Gitlab attr_writer :logger def logger - @logger ||= Logger.new(STDOUT) + @logger ||= Logger.new($stdout) end end @@ -67,7 +67,7 @@ module Gitlab def log_info(details) details = base_log_data.merge(details) - details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, STDOUT) + details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, $stdout) Measuring.logger.info(details) end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index b1ccdcb1df0..4ea5b5a87de 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -42,9 +42,6 @@ module Gitlab FALLBACK = -1 HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 - ALL_TIME_TIME_FRAME_NAME = "all" - SEVEN_DAYS_TIME_FRAME_NAME = "7d" - TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" MAX_BUCKET_SIZE = 100 def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) @@ -227,7 +224,7 @@ module Gitlab } # rubocop: disable CodeReuse/ActiveRecord - JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| counts = services.group_by do |service| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e9905bae985..0f33c3aa68e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -270,7 +270,7 @@ module Gitlab prefix: metadata['ArchivePrefix'], format: format, path: path.presence || "", - include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) + include_lfs_blobs: true ).to_proto ) } diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 34e3be2320b..c917debd3d9 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -13,10 +13,6 @@ module GoogleApi LEAST_TOKEN_LIFE_TIME = 10.minutes CLUSTER_MASTER_AUTH_USERNAME = 'admin' CLUSTER_IPV4_CIDR_BLOCK = '/16' - # Don't upgrade to > 1.18 before we move away from Basic Auth - # See issue: https://gitlab.com/gitlab-org/gitlab/-/issues/331582 - # Possible solution: https://gitlab.com/groups/gitlab-org/-/epics/6049 - GKE_VERSION = '1.18' CLUSTER_OAUTH_SCOPES = [ "https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/logging.write", @@ -94,13 +90,11 @@ module GoogleApi cluster: { name: cluster_name, initial_node_count: cluster_size, - initial_cluster_version: GKE_VERSION, node_config: { machine_type: machine_type, oauth_scopes: CLUSTER_OAUTH_SCOPES }, master_auth: { - username: CLUSTER_MASTER_AUTH_USERNAME, client_certificate_config: { issue_client_certificate: true } diff --git a/lib/mattermost/error.rb b/lib/mattermost.rb index 054bd5457bd..054bd5457bd 100644 --- a/lib/mattermost/error.rb +++ b/lib/mattermost.rb diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index 7fb959a149c..a5c1f788c68 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Mattermost - ClientError = Class.new(Mattermost::Error) + ClientError = Class.new(::Mattermost::Error) class Client attr_reader :user @@ -11,7 +11,7 @@ module Mattermost end def with_session(&blk) - Mattermost::Session.new(user).with_session(&blk) + ::Mattermost::Session.new(user).with_session(&blk) end private @@ -52,12 +52,12 @@ module Mattermost json_response = Gitlab::Json.parse(response.body, legacy_mode: true) unless response.success? - raise Mattermost::ClientError, json_response['message'] || 'Undefined error' + raise ::Mattermost::ClientError, json_response['message'] || 'Undefined error' end json_response rescue JSON::JSONError - raise Mattermost::ClientError, 'Cannot parse response' + raise ::Mattermost::ClientError, 'Cannot parse response' end end end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 523d82f9161..9374c5c8f8f 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true module Mattermost - class NoSessionError < Mattermost::Error + class NoSessionError < ::Mattermost::Error def message 'No session could be set up, is Mattermost configured with Single Sign On?' end end - ConnectionError = Class.new(Mattermost::Error) + ConnectionError = Class.new(::Mattermost::Error) # This class' prime objective is to obtain a session token on a Mattermost # instance with SSO configured where this GitLab instance is the provider. @@ -42,7 +42,7 @@ module Mattermost yield self rescue Errno::ECONNREFUSED => e Gitlab::AppLogger.error(e.message + "\n" + e.backtrace.join("\n")) - raise Mattermost::NoSessionError + raise ::Mattermost::NoSessionError ensure destroy end @@ -100,11 +100,11 @@ module Mattermost end def create - raise Mattermost::NoSessionError unless oauth_uri - raise Mattermost::NoSessionError unless token_uri + raise ::Mattermost::NoSessionError unless oauth_uri + raise ::Mattermost::NoSessionError unless token_uri @token = request_token - raise Mattermost::NoSessionError unless @token + raise ::Mattermost::NoSessionError unless @token @headers = { Authorization: "Bearer #{@token}" @@ -174,9 +174,9 @@ module Mattermost def handle_exceptions yield rescue Gitlab::HTTP::Error => e - raise Mattermost::ConnectionError, e.message + raise ::Mattermost::ConnectionError, e.message rescue Errno::ECONNREFUSED => e - raise Mattermost::ConnectionError, e.message + raise ::Mattermost::ConnectionError, e.message end def parse_cookie(response) diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index 39005f56dcb..299e3eeb953 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -32,7 +32,7 @@ module MicrosoftTeams result['title'] = title result['summary'] = summary - result['sections'] << MicrosoftTeams::Activity.new(**activity).prepare + result['sections'] << ::MicrosoftTeams::Activity.new(**activity).prepare unless attachments.blank? result['sections'] << { text: attachments } diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 774e4768597..8e1200338c2 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -43,6 +43,11 @@ module Peek count[item[:transaction]] ||= 0 count[item[:transaction]] += 1 end + + if ::Gitlab::Database::LoadBalancing.enable? + count[item[:db_role]] ||= 0 + count[item[:db_role]] += 1 + end end def setup_subscribers @@ -60,11 +65,19 @@ module Peek sql: data[:sql].strip, backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), cached: data[:cached] ? 'Cached' : '', - transaction: data[:connection].transaction_open? ? 'In a transaction' : '' + transaction: data[:connection].transaction_open? ? 'In a transaction' : '', + db_role: db_role(data) } end + + def db_role(data) + return unless ::Gitlab::Database::LoadBalancing.enable? + + role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(data[:connection]) || + ::Gitlab::Database::LoadBalancing::ROLE_UNKNOWN + + role.to_s.capitalize + end end end end - -Peek::Views::ActiveRecord.prepend_mod_with('Peek::Views::ActiveRecord') diff --git a/lib/peek/views/memory.rb b/lib/peek/views/memory.rb new file mode 100644 index 00000000000..399474dedf1 --- /dev/null +++ b/lib/peek/views/memory.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Peek + module Views + class Memory < View + MEM_TOTAL_LABEL = 'Total' + MEM_OBJECTS_LABEL = 'Objects allocated' + MEM_MALLOCS_LABEL = 'Allocator calls' + MEM_BYTES_LABEL = 'Large allocations' + + def initialize(options = {}) + super + + @thread_memory = {} + end + + def results + return thread_memory if thread_memory.empty? + + { + calls: byte_string(thread_memory[:mem_total_bytes]), + summary: { + MEM_OBJECTS_LABEL => number_string(thread_memory[:mem_objects]), + MEM_MALLOCS_LABEL => number_string(thread_memory[:mem_mallocs]), + MEM_BYTES_LABEL => byte_string(thread_memory[:mem_bytes]) + }, + details: [ + { + item_header: MEM_TOTAL_LABEL, + item_content: "Total memory use of this request. This includes both occupancy of existing heap slots " \ + "as well as newly allocated memory due to large objects. Not adjusted for freed memory. " \ + "Lower is better." + }, + { + item_header: MEM_OBJECTS_LABEL, + item_content: "Total number of objects allocated by the Ruby VM during this request. " \ + "Not adjusted for objects that were freed again. Lower is better." + }, + { + item_header: MEM_MALLOCS_LABEL, + item_content: "Total number of times Ruby had to call `malloc`, the C memory allocator. " \ + "This is necessary for objects that are too large to fit into a 40 Byte slot in Ruby's managed heap. " \ + "Lower is better." + }, + { + item_header: MEM_BYTES_LABEL, + item_content: "Memory allocated for objects that did not fit into a heap slot. " \ + "Not adjusted for memory that was freed again. Lower is better." + } + ] + } + end + + private + + attr_reader :thread_memory + + def setup_subscribers + subscribe 'process_action.action_controller' do + # Ensure that Peek will see memory instrumentation in `results` by triggering it when + # a request is done processing. Peek itself hooks into the same notification: + # https://github.com/peek/peek/blob/master/lib/peek/railtie.rb + Gitlab::InstrumentationHelper.instrument_thread_memory_allocations(thread_memory) + end + end + + def byte_string(bytes) + ActiveSupport::NumberHelper.number_to_human_size(bytes) + end + + def number_string(num) + ActiveSupport::NumberHelper.number_to_human(num, units: { thousand: 'k', million: 'M', billion: 'B' }) + end + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index 32beeb0d31e..d2563b4c806 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -7,8 +7,6 @@ module Prometheus def worker_id if Gitlab::Runtime.sidekiq? sidekiq_worker_id - elsif Gitlab::Runtime.unicorn? - unicorn_worker_id elsif Gitlab::Runtime.puma? puma_worker_id else @@ -26,16 +24,6 @@ module Prometheus end end - def unicorn_worker_id - if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) - "unicorn_#{matches[1]}" - elsif process_name =~ /unicorn/ - "unicorn_master" - else - unknown_process_id - end - end - def puma_worker_id if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) "puma_#{matches[1]}" diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb index 133afcb52ae..dff55eead2f 100644 --- a/lib/release_highlights/validator/entry.rb +++ b/lib/release_highlights/validator/entry.rb @@ -46,7 +46,10 @@ module ReleaseHighlights def add_line_numbers_to_errors! errors.messages.each do |attribute, messages| - messages.map! { |m| "#{m} (line #{line_number_for(attribute)})" } + extended_messages = messages.map { |m| "#{m} (line #{line_number_for(attribute)})" } + + errors.delete(attribute) + extended_messages.each { |extended_message| errors.add(attribute, extended_message) } end end diff --git a/lib/security/ci_configuration/base_build_action.rb b/lib/security/ci_configuration/base_build_action.rb index b169d780cad..e7a1b4770b9 100644 --- a/lib/security/ci_configuration/base_build_action.rb +++ b/lib/security/ci_configuration/base_build_action.rb @@ -42,7 +42,7 @@ module Security # SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings # Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings # Note that environment variables can be set in several places - # See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables + # See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence YAML end end diff --git a/lib/security/ci_configuration/sast_build_action.rb b/lib/security/ci_configuration/sast_build_action.rb index 23dd4bd6d14..3fa5e9c7177 100644 --- a/lib/security/ci_configuration/sast_build_action.rb +++ b/lib/security/ci_configuration/sast_build_action.rb @@ -3,8 +3,6 @@ module Security module CiConfiguration class SastBuildAction < BaseBuildAction - SAST_DEFAULT_ANALYZERS = 'bandit, brakeman, eslint, flawfinder, gosec, kubesec, nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow, spotbugs' - def initialize(auto_devops_enabled, params, existing_gitlab_ci_content) super(auto_devops_enabled, existing_gitlab_ci_content) @variables = variables(params) @@ -114,7 +112,6 @@ module Security def sast_variables %w( - SAST_ANALYZER_IMAGE_TAG SAST_EXCLUDED_PATHS SEARCH_MAX_DEPTH SAST_EXCLUDED_ANALYZERS diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb index 1ed5d5dc3f5..6564f53d2da 100644 --- a/lib/serializers/json.rb +++ b/lib/serializers/json.rb @@ -2,7 +2,7 @@ module Serializers # Make the resulting hash have deep indifferent access - class JSON + class Json class << self def dump(obj) obj diff --git a/lib/sidebars/concerns/container_with_html_options.rb b/lib/sidebars/concerns/container_with_html_options.rb index 873cb5b0de9..79dddd309b5 100644 --- a/lib/sidebars/concerns/container_with_html_options.rb +++ b/lib/sidebars/concerns/container_with_html_options.rb @@ -38,6 +38,16 @@ module Sidebars # in the helper method that sets the active class # on each element. def nav_link_html_options + { + data: { + track_label: self.class.name.demodulize.underscore + } + }.deep_merge(extra_nav_link_html_options) + end + + # Classes should mostly override this method + # and not `nav_link_html_options`. + def extra_nav_link_html_options {} end diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index d81e413f4a9..dcdc130b0d7 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -83,6 +83,16 @@ module Sidebars insert_element_after(@items, after_item, new_item) end + override :container_html_options + def container_html_options + super.tap do |html_options| + # Flagging menus that can be rendered and with renderable menu items + if render? && has_renderable_items? + html_options[:class] = [*html_options[:class], 'has-sub-items'].join(' ') + end + end + end + private override :index_of diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb index b0a12e769dc..1375f9fffca 100644 --- a/lib/sidebars/menu_item.rb +++ b/lib/sidebars/menu_item.rb @@ -22,5 +22,13 @@ module Sidebars def render? true end + + def nav_link_html_options + { + data: { + track_label: item_id + } + } + end end end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 75b6cae295f..8cf7abc613c 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -6,7 +6,7 @@ module Sidebars class InfrastructureMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false if Feature.disabled?(:sidebar_refactor, context.current_user) + return false if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) return false unless context.project.feature_available?(:operations, context.current_user) add_item(kubernetes_menu_item) diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb index 9840f644179..79603803b8f 100644 --- a/lib/sidebars/projects/menus/issues_menu.rb +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -98,7 +98,7 @@ module Sidebars end def labels_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user) + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) return ::Sidebars::NilMenuItem.new(item_id: :labels) end diff --git a/lib/sidebars/projects/menus/labels_menu.rb b/lib/sidebars/projects/menus/labels_menu.rb index 12cf0444994..7cb28ababdb 100644 --- a/lib/sidebars/projects/menus/labels_menu.rb +++ b/lib/sidebars/projects/menus/labels_menu.rb @@ -40,7 +40,7 @@ module Sidebars override :render? def render? - return false if Feature.enabled?(:sidebar_refactor, context.current_user) + return false if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) can?(context.current_user, :read_label, context.project) && !context.project.issues_enabled? end diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb index e3fcd8f25d5..f29f4a6eed6 100644 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -35,14 +35,13 @@ module Sidebars end end - override :extra_container_html_options - def nav_link_html_options + override :extra_nav_link_html_options + def extra_nav_link_html_options { class: 'home', data: { - track_action: 'click_menu', - track_property: context.learn_gitlab_experiment_tracking_category, - track_label: 'learn_gitlab' + track_label: 'learn_gitlab', + track_property: context.learn_gitlab_experiment_tracking_category } } end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 18c990d0e1f..8ebdacc7c7e 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -139,7 +139,7 @@ module Sidebars end def serverless_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user) || + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || !can?(context.current_user, :read_cluster, context.project) return ::Sidebars::NilMenuItem.new(item_id: :serverless) end @@ -153,7 +153,7 @@ module Sidebars end def terraform_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user) || + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || !can?(context.current_user, :read_terraform_state, context.project) return ::Sidebars::NilMenuItem.new(item_id: :terraform) end @@ -167,7 +167,7 @@ module Sidebars end def kubernetes_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user) || + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || !can?(context.current_user, :read_cluster, context.project) return ::Sidebars::NilMenuItem.new(item_id: :kubernetes) end diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 7087916bb04..27e318d73c5 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -58,7 +58,7 @@ module Sidebars end def infrastructure_registry_menu_item - if Feature.disabled?(:infrastructure_registry_page, context.current_user) + if Feature.disabled?(:infrastructure_registry_page, context.current_user, default_enabled: :yaml) return ::Sidebars::NilMenuItem.new(item_id: :infrastructure_registry) end diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb index cbb34714087..c148e7cf931 100644 --- a/lib/sidebars/projects/menus/project_information_menu.rb +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -17,24 +17,26 @@ module Sidebars override :link def link - project_path(context.project) + renderable_items.first.link end override :extra_container_html_options def extra_container_html_options - { - class: 'shortcuts-project rspec-project-link' - } + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + { class: 'shortcuts-project-information' } + else + { class: 'shortcuts-project rspec-project-link' } + end end - override :nav_link_html_options - def nav_link_html_options + override :extra_nav_link_html_options + def extra_nav_link_html_options { class: 'home' } end override :title def title - if Feature.enabled?(:sidebar_refactor, context.current_user) + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) _('Project information') else _('Project overview') @@ -43,24 +45,17 @@ module Sidebars override :sprite_icon def sprite_icon - if Feature.enabled?(:sidebar_refactor, context.current_user) + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) 'project' else 'home' end end - override :active_routes - def active_routes - return {} if Feature.disabled?(:sidebar_refactor, context.current_user) - - { path: 'projects#show' } - end - private def details_menu_item - return if Feature.enabled?(:sidebar_refactor, context.current_user) + return if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ::Sidebars::MenuItem.new( title: _('Details'), @@ -103,7 +98,7 @@ module Sidebars end def labels_menu_item - if Feature.disabled?(:sidebar_refactor, context.current_user) + if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) return ::Sidebars::NilMenuItem.new(item_id: :labels) end diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb index 1d1cf11b271..1cd0218d4ac 100644 --- a/lib/sidebars/projects/menus/scope_menu.rb +++ b/lib/sidebars/projects/menus/scope_menu.rb @@ -13,6 +13,32 @@ module Sidebars def title context.project.name end + + override :active_routes + def active_routes + { path: 'projects#show' } + end + + override :extra_container_html_options + def extra_container_html_options + return {} if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + + { + class: 'shortcuts-project rspec-project-link' + } + end + + override :extra_nav_link_html_options + def extra_nav_link_html_options + return {} if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + + { class: 'context-header' } + end + + override :render? + def render? + true + end end end end diff --git a/lib/sidebars/projects/menus/security_compliance_menu.rb b/lib/sidebars/projects/menus/security_compliance_menu.rb index 6c9fb8312bd..5616b466560 100644 --- a/lib/sidebars/projects/menus/security_compliance_menu.rb +++ b/lib/sidebars/projects/menus/security_compliance_menu.rb @@ -17,7 +17,7 @@ module Sidebars override :link def link - project_security_configuration_path(context.project) + renderable_items.first&.link end override :title @@ -33,18 +33,16 @@ module Sidebars private def configuration_menu_item - strong_memoize(:configuration_menu_item) do - unless render_configuration_menu_item? - next ::Sidebars::NilMenuItem.new(item_id: :configuration) - end - - ::Sidebars::MenuItem.new( - title: _('Configuration'), - link: project_security_configuration_path(context.project), - active_routes: { path: configuration_menu_item_paths }, - item_id: :configuration - ) + unless render_configuration_menu_item? + return ::Sidebars::NilMenuItem.new(item_id: :configuration) end + + ::Sidebars::MenuItem.new( + title: _('Configuration'), + link: project_security_configuration_path(context.project), + active_routes: { path: configuration_menu_item_paths }, + item_id: :configuration + ) end def render_configuration_menu_item? diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 4ea6f5e298a..c9d7e736b21 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -136,7 +136,7 @@ module Sidebars def packages_and_registries_menu_item if !Gitlab.config.registry.enabled || - Feature.disabled?(:sidebar_refactor, context.current_user) || + Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || !can?(context.current_user, :destroy_container_image, context.project) return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 9bf8fe28120..ac47c5be1e8 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -26,17 +26,6 @@ ### Environment variables RAILS_ENV=${RAILS_ENV:-'production'} SIDEKIQ_WORKERS=${SIDEKIQ_WORKERS:-1} -USE_WEB_SERVER=${USE_WEB_SERVER:-'puma'} - -case "${USE_WEB_SERVER}" in - puma|unicorn) - use_web_server="$USE_WEB_SERVER" - ;; - *) - echo "Unsupported web server '${USE_WEB_SERVER}' (Allowed: 'puma', 'unicorn')" 1>&2 - exit 1 - ;; -esac # Script variable names should be lower-case not to conflict with # internal /bin/sh variables such as PATH, EDITOR or SHELL. @@ -45,7 +34,7 @@ app_root="/home/$app_user/gitlab" pid_path="$app_root/tmp/pids" socket_path="$app_root/tmp/sockets" rails_socket="$socket_path/gitlab.socket" -web_server_pid_path="$pid_path/$use_web_server.pid" +web_server_pid_path="$pid_path/puma.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd) @@ -270,7 +259,7 @@ start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab web server ($use_web_server)" + echo "Starting GitLab web server" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -295,7 +284,7 @@ start_gitlab() { # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web start + RAILS_ENV=$RAILS_ENV bin/web start fi # If sidekiq is already running, don't start it again. @@ -357,7 +346,7 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab web server" - RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web stop + RAILS_ENV=$RAILS_ENV bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -461,7 +450,7 @@ reload_gitlab(){ exit 1 fi printf "Reloading GitLab web server configuration... " - RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web reload + RAILS_ENV=$RAILS_ENV bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1b499467ad6..53bebe55fa3 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -5,9 +5,6 @@ # Normal values are "production", "test" and "development". RAILS_ENV="production" -# Uncomment the line below to enable the Unicorn web server instead of Puma. -# use_web_server="unicorn" - # app_user defines the user that GitLab is run as. # The default is "git". app_user="git" @@ -43,7 +40,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # socket. To listen on TCP connections (needed by Apache) change to: # '-listenNetwork tcp -listenAddr 127.0.0.1:8181' # -# The -authBackend setting tells gitlab-workhorse where it can reach Unicorn. +# The -authBackend setting tells gitlab-workhorse where it can reach the GitLab Rails application. # For relative URL support change to: # '-authBackend http://127.0.0.1/8080/gitlab' # Read more in http://doc.gitlab.com/ce/install/relative_url.html diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb index e72d8b6b04d..d907c041ad8 100644 --- a/lib/system_check/app/redis_version_check.rb +++ b/lib/system_check/app/redis_version_check.rb @@ -5,9 +5,9 @@ require 'redis' module SystemCheck module App class RedisVersionCheck < SystemCheck::BaseCheck - # Redis 4.x will be deprecated - # https://gitlab.com/gitlab-org/gitlab/-/issues/327197 - MIN_REDIS_VERSION = '4.0.0' + # Redis 5.x will be deprecated + # https://gitlab.com/gitlab-org/gitlab/-/issues/331468 + MIN_REDIS_VERSION = '5.0.0' RECOMMENDED_REDIS_VERSION = '5.0.0' set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?" diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index 61719abc991..e02ae81dc98 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -52,7 +52,7 @@ module SystemCheck def load_config erb = ERB.new(File.read(mail_room_config_path)) erb.filename = mail_room_config_path - config_file = YAML.safe_load(erb.result) + config_file = YAML.safe_load(erb.result, permitted_classes: [Symbol]) config_file[:mailboxes] end diff --git a/lib/tasks/file_hooks.rake b/lib/tasks/file_hooks.rake index a892d36b48e..5eb49808eff 100644 --- a/lib/tasks/file_hooks.rake +++ b/lib/tasks/file_hooks.rake @@ -3,14 +3,9 @@ namespace :file_hooks do desc 'Validate existing file hooks' task validate: :environment do - puts 'Validating file hooks from /file_hooks and /plugins directories' + puts 'Validating file hooks from /file_hooks directories' Gitlab::FileHook.files.each do |file| - if File.dirname(file).ends_with?('plugins') - puts 'DEPRECATED: /plugins directory is deprecated and will be removed in 14.0. ' \ - 'Please move your files into /file_hooks directory.' - end - success, message = Gitlab::FileHook.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) if success diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 4c312ea492b..084e7c78906 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -7,7 +7,7 @@ desc 'GitLab | Artifacts | Migrate files for artifacts to comply with new storag namespace :gitlab do namespace :artifacts do task migrate: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) @@ -19,7 +19,7 @@ namespace :gitlab do end task migrate_to_local: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index c53ef8382b8..5b17a8c185a 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -109,7 +109,7 @@ namespace :gitlab do puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) exit 1 else - Backup::Repositories.new(progress).dump( + Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump( max_concurrency: max_concurrency, max_storage_concurrency: max_storage_concurrency ) @@ -119,7 +119,7 @@ namespace :gitlab do task restore: :gitlab_environment do puts_time "Restoring repositories ...".color(:blue) - Backup::Repositories.new(progress).restore + Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore puts_time "done".color(:green) end end @@ -294,6 +294,14 @@ namespace :gitlab do $stdout end end + + def repository_backup_strategy + if Feature.enabled?(:gitaly_backup) + Backup::GitalyBackup.new(progress) + else + Backup::GitalyRpcBackup.new(progress) + end + end end # namespace end: backup end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 6c3a7a77e0e..0cd4ab354c9 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -92,9 +92,9 @@ namespace :gitlab do task orphan_lfs_files: :gitlab_environment do warn_user_is_not_gitlab - removed_files = RemoveUnreferencedLfsObjectsWorker.new.perform + number_of_removed_files = RemoveUnreferencedLfsObjectsWorker.new.perform - logger.info "Removed unreferenced LFS files: #{removed_files.count}".color(:green) + logger.info "Removed unreferenced LFS files: #{number_of_removed_files}".color(:green) end namespace :sessions do @@ -178,7 +178,7 @@ namespace :gitlab do return @logger if defined?(@logger) @logger = if Rails.env.development? || Rails.env.production? - Logger.new(STDOUT).tap do |stdout_logger| + Logger.new($stdout).tap do |stdout_logger| stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger)) stdout_logger.level = debug? ? Logger::DEBUG : Logger::INFO end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index bbfdf598e42..ee986f4c503 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -129,16 +129,31 @@ namespace :gitlab do end # Inform Rake that custom tasks should be run every time rake db:structure:dump is run + # + # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump Rake::Task['db:structure:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke Rake::Task['gitlab:db:dump_custom_structure'].invoke end + # Inform Rake that custom tasks should be run every time rake db:schema:dump is run + Rake::Task['db:schema:dump'].enhance do + Rake::Task['gitlab:db:clean_structure_sql'].invoke + Rake::Task['gitlab:db:dump_custom_structure'].invoke + end + # Inform Rake that custom tasks should be run every time rake db:structure:load is run + # + # Rails 6.1 deprecates db:structure:load in favor of db:schema:load Rake::Task['db:structure:load'].enhance do Rake::Task['gitlab:db:load_custom_structure'].invoke end + # Inform Rake that custom tasks should be run every time rake db:schema:load is run + Rake::Task['db:schema:load'].enhance do + Rake::Task['gitlab:db:load_custom_structure'].invoke + end + desc 'Create missing dynamic database partitions' task :create_dynamic_partitions do Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions @@ -159,10 +174,16 @@ namespace :gitlab do # # Other than that it's helpful to create partitions early when bootstrapping # a new installation. + # + # Rails 6.1 deprecates db:structure:load in favor of db:schema:load Rake::Task['db:structure:load'].enhance do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end + Rake::Task['db:schema:load'].enhance do + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + end + # During testing, db:test:load restores the database schema from scratch # which does not include dynamic partitions. We cannot rely on application # initializers here as the application can continue to run while @@ -188,7 +209,7 @@ namespace :gitlab do raise "Index not found or not supported: #{args[:index_name]}" if indexes.empty? end - ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) Gitlab::Database::Reindexing.perform(indexes) rescue StandardError => e @@ -219,9 +240,7 @@ namespace :gitlab do desc 'Run migrations with instrumentation' task migration_testing: :environment do result_dir = Gitlab::Database::Migrations::Instrumentation::RESULT_DIR - raise "Directory exists already, won't overwrite: #{result_dir}" if File.exist?(result_dir) - - Dir.mkdir(result_dir) + FileUtils.mkdir_p(result_dir) verbose_was = ActiveRecord::Migration.verbose ActiveRecord::Migration.verbose = true diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 0c8e0755348..990ff723eeb 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -1,8 +1,11 @@ # frozen_string_literal: true require 'date' require 'pathname' +require "yaml" +# # https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page +# namespace :gitlab do namespace :docs do desc 'GitLab | Docs | Create a doc redirect' @@ -11,14 +14,14 @@ namespace :gitlab do old_path = args.old_path else puts '=> Enter the path of the OLD file:' - old_path = STDIN.gets.chomp + old_path = $stdin.gets.chomp end if args.new_path new_path = args.new_path else puts '=> Enter the path of the NEW file:' - new_path = STDIN.gets.chomp + new_path = $stdin.gets.chomp end # @@ -38,13 +41,14 @@ namespace :gitlab do # - If this is an external URL, move the date 1 year later. # - If this is a relative URL, move the date 3 months later. # - date = Time.now.utc.strftime('%Y-%m-%d') - date = new_path.start_with?('http') ? Date.parse(date) >> 12 : Date.parse(date) >> 3 + today = Time.now.utc.to_date + date = new_path.start_with?('http') ? today >> 12 : today >> 3 puts "=> Creating new redirect from #{old_path} to #{new_path}" File.open(old_path, 'w') do |post| post.puts '---' post.puts "redirect_to: '#{new_path}'" + post.puts "remove_date: '#{date}'" post.puts '---' post.puts post.puts "This file was moved to [another location](#{new_path})." @@ -53,5 +57,68 @@ namespace :gitlab do post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->" end end + + desc 'GitLab | Docs | Clean up old redirects' + task :clean_redirects do + # + # Calculate new path from the redirect URL. + # + # If the redirect is not a full URL: + # 1. Create a new Pathname of the file + # 2. Use dirname to get all but the last component of the path + # 3. Join with the redirect_to entry + # 4. Substitute: + # - '.md' => '.html' + # - 'doc/' => '/ee/' + # + # If the redirect URL is a full URL pointing to the Docs site + # (cross-linking among the 4 products), remove the FQDN prefix: + # + # From : https://docs.gitlab.com/ee/install/requirements.html + # To : /ee/install/requirements.html + # + def new_path(redirect, filename) + if !redirect.start_with?('http') + Pathname.new(filename).dirname.join(redirect).to_s.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') + elsif redirect.start_with?('https://docs.gitlab.com') + redirect.gsub('https://docs.gitlab.com', '') + else + redirect + end + end + + today = Time.now.utc.to_date + + # + # Find the files to be deleted. + # Exclude 'doc/development/documentation/index.md' because it + # contains an example of the YAML front matter. + # + files_to_be_deleted = `grep -Ir 'remove_date:' doc | grep -v doc/development/documentation/index.md | cut -d ":" -f 1`.split("\n") + + # + # Iterate over the files to be deleted and print the needed + # YAML entries for the Docs site redirects. + # + files_to_be_deleted.each do |filename| + frontmatter = YAML.safe_load(File.read(filename)) + remove_date = Date.parse(frontmatter['remove_date']) + old_path = filename.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') + + # + # Check if the removal date is before today, and delete the file and + # print the content to be pasted in + # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/redirects.yaml. + # The remove_date of redirects.yaml should be nine months in the future. + # To not be confused with the remove_date of the Markdown page. + # + next unless remove_date < today + + File.delete(filename) if File.exist?(filename) + puts " - from: #{old_path}" + puts " to: #{new_path(frontmatter['redirect_to'], filename)}" + puts " remove_date: #{remove_date >> 9}" + end + end end end diff --git a/lib/tasks/gitlab/doctor/secrets.rake b/lib/tasks/gitlab/doctor/secrets.rake index 6e3f474312c..29f0f36c705 100644 --- a/lib/tasks/gitlab/doctor/secrets.rake +++ b/lib/tasks/gitlab/doctor/secrets.rake @@ -4,7 +4,7 @@ namespace :gitlab do namespace :doctor do desc "GitLab | Check if the database encrypted values can be decrypted using current secrets" task secrets: :gitlab_environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::DEBUG : Logger::INFO diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index 27bba6aa307..b405cbd3f68 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -3,11 +3,12 @@ return if Rails.env.production? require 'graphql/rake_task' +require_relative '../../../tooling/graphql/docs/renderer' namespace :gitlab do OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference") TEMP_SCHEMA_DIR = Rails.root.join('tmp/tests/graphql') - TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/' + TEMPLATES_DIR = 'tooling/graphql/docs/templates/' # Make all feature flags enabled so that all feature flag # controlled fields are considered visible and are output. @@ -110,7 +111,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Generate GraphQL docs' task compile_docs: [:environment, :enable_feature_flags] do - renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options) + renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, render_options) renderer.write @@ -119,7 +120,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Check if GraphQL docs are up to date' task check_docs: [:environment, :enable_feature_flags] do - renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options) + renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, render_options) doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md')) diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake index 3b2834c0008..4da22e686ef 100644 --- a/lib/tasks/gitlab/ldap.rake +++ b/lib/tasks/gitlab/ldap.rake @@ -42,7 +42,7 @@ namespace :gitlab do namespace :secret do desc 'GitLab | LDAP | Secret | Write LDAP secrets' task write: [:environment] do - content = STDIN.tty? ? STDIN.gets : STDIN.read + content = $stdin.tty? ? $stdin.gets : $stdin.read Gitlab::EncryptedLdapCommand.write(content) end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index a173de7c5c7..47f9e1dfb32 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -6,7 +6,7 @@ desc "GitLab | LFS | Migrate LFS objects to remote storage" namespace :gitlab do namespace :lfs do task migrate: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Starting transfer of LFS files to object storage') LfsObject.with_files_stored_locally @@ -20,7 +20,7 @@ namespace :gitlab do end task migrate_to_local: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Starting transfer of LFS files to local storage') LfsObject.with_files_stored_remotely diff --git a/lib/tasks/gitlab/packages/composer.rake b/lib/tasks/gitlab/packages/composer.rake index c9bccfe9384..97f1da0ff63 100644 --- a/lib/tasks/gitlab/packages/composer.rake +++ b/lib/tasks/gitlab/packages/composer.rake @@ -6,7 +6,7 @@ desc "GitLab | Packages | Build composer cache" namespace :gitlab do namespace :packages do task build_composer_cache: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Starting to build composer cache files') ::Packages::Package.composer.find_in_batches do |packages| diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index d24535d85b6..a5b801ff62d 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -14,7 +14,7 @@ namespace :gitlab do end task generate_counts: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Building list of package events...') path = Gitlab::UsageDataCounters::PackageEventCounter::KNOWN_EVENTS_PATH @@ -26,7 +26,7 @@ namespace :gitlab do end task generate_unique: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Building list of package events...') path = File.join(File.dirname(Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml') diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake index febc3e7fa2d..1c28f4308a2 100644 --- a/lib/tasks/gitlab/packages/migrate.rake +++ b/lib/tasks/gitlab/packages/migrate.rake @@ -6,7 +6,7 @@ desc "GitLab | Packages | Migrate packages files to remote storage" namespace :gitlab do namespace :packages do task migrate: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Starting transfer of package files to object storage') unless ::Packages::PackageFileUploader.object_store_enabled? diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index 684d62d1367..c3828e7eba4 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -35,7 +35,7 @@ namespace :gitlab do end def logger - @logger ||= Logger.new(STDOUT) + @logger ||= Logger.new($stdout) end def migration_threads @@ -60,7 +60,7 @@ namespace :gitlab do namespace :deployments do task migrate_to_object_storage: :gitlab_environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) @@ -72,7 +72,7 @@ namespace :gitlab do end task migrate_to_local: :gitlab_environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 31bd80e78d4..705519d1741 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -40,7 +40,7 @@ namespace :gitlab do end # If there are any clients connected to the DB, PostgreSQL won't let - # you drop the database. It's possible that Sidekiq, Unicorn, or + # you drop the database. It's possible that Sidekiq, Puma, or # some other client will be hanging onto a connection, preventing # the DROP DATABASE from working. To workaround this problem, this # method terminates all the connections so that a subsequent DROP diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index ede6b6af80b..6fa39a26488 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -96,8 +96,12 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' task legacy_projects: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository)) + # Required to prevent Docker upgrade to 14.0 if there data on legacy storage + # See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698 + wait_until_database_is_ready do + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository)) + end end desc 'Gitlab | Storage | List existing projects using Legacy Storage' @@ -135,8 +139,12 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' task legacy_attachments: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation) + # Required to prevent Docker upgrade to 14.0 if there data on legacy storage + # See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698 + wait_until_database_is_ready do + helper = Gitlab::HashedStorage::RakeHelper + helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation) + end end desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' @@ -156,5 +164,23 @@ namespace :gitlab do helper = Gitlab::HashedStorage::RakeHelper helper.attachments_list('attachments using Hashed Storage', helper.hashed_attachments_relation) end + + def wait_until_database_is_ready + attempts = (ENV['MAX_DATABASE_CONNECTION_CHECKS'] || 1).to_i + inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f + + attempts.to_i.times do + unless Gitlab::Database.exists? + puts "Waiting until database is ready before continuing...".color(:yellow) + sleep inverval + end + end + + yield + rescue ActiveRecord::ConnectionNotEstablished => ex + puts "Failed to connect to the database...".color(:red) + puts "Error: #{ex}" + exit 1 + end end end diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake index 2bf9ec9537a..99e33011cf5 100644 --- a/lib/tasks/gitlab/terraform/migrate.rake +++ b/lib/tasks/gitlab/terraform/migrate.rake @@ -6,7 +6,7 @@ desc "GitLab | Terraform | Migrate Terraform states to remote storage" namespace :gitlab do namespace :terraform_states do task migrate: :environment do - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) logger.info('Starting transfer of Terraform states to object storage') begin diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 6052ff90341..80290f95e8e 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -16,7 +16,7 @@ namespace :gitlab do # category to object storage desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| - Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage + Gitlab::Uploads::MigrationHelper.new(args, Logger.new($stdout)).migrate_to_remote_storage end namespace :migrate_to_local do @@ -31,7 +31,7 @@ namespace :gitlab do desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| - Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage + Gitlab::Uploads::MigrationHelper.new(args, Logger.new($stdout)).migrate_to_local_storage end end end diff --git a/lib/tasks/gitlab/uploads/sanitize.rake b/lib/tasks/gitlab/uploads/sanitize.rake index eec423cbb8b..40f6a7bb67d 100644 --- a/lib/tasks/gitlab/uploads/sanitize.rake +++ b/lib/tasks/gitlab/uploads/sanitize.rake @@ -8,7 +8,7 @@ namespace :gitlab do args.with_defaults(dry_run: 'true') args.with_defaults(sleep_time: 0.3) - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) sanitizer = Gitlab::Sanitizers::Exif.new(logger: logger) sanitizer.batch_clean(start_id: args.start_id, stop_id: args.stop_id, diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake index de878a3d093..d3c63fa8514 100644 --- a/lib/tasks/gitlab/x509/update.rake +++ b/lib/tasks/gitlab/x509/update.rake @@ -10,7 +10,7 @@ namespace :gitlab do end def update_certificates - logger = Logger.new(STDOUT) + logger = Logger.new($stdout) unless X509CommitSignature.exists? logger.info("Unable to find any x509 commit signatures. Exiting.") diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 633beb132d8..b7a5cbe44b9 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -38,7 +38,7 @@ class GithubImport puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) - STDIN.getch + $stdin.getch puts 'Starting the import (this could take a while)'.color(:green) end @@ -131,7 +131,7 @@ class GithubRepos end def repo_id - @repo_id ||= STDIN.gets.chomp.to_i + @repo_id ||= $stdin.gets.chomp.to_i end def repos diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 74baa11c314..ff14ab51b49 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -19,7 +19,7 @@ namespace :tokens do def reset_all_users_token(reset_token_method) TmpUser.find_in_batches do |batch| puts "Processing batch starting with user ID: #{batch.first.id}" - STDOUT.flush + $stdout.flush batch.each(&reset_token_method) end |