diff options
Diffstat (limited to 'lib')
213 files changed, 4013 insertions, 1899 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index ada0da28749..725dddead70 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -30,7 +30,7 @@ module API ] allow_access_with_scope :api - allow_access_with_scope :read_api, if: -> (request) { request.get? } + allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? } prefix :api version 'v3', using: :path do @@ -123,13 +123,32 @@ module API format :json formatter :json, Gitlab::Json::GrapeFormatter + content_type :json, 'application/json' + # Remove the `text/plain+deprecated` with `api_always_use_application_json` feature flag # There is a small chance some users depend on the old behavior. # We this change under a feature flag to see if affects GitLab.com users. - if Gitlab::Database.cached_table_exists?('features') && Feature.enabled?(:api_json_content_type) - content_type :json, 'application/json' - else - content_type :txt, 'text/plain' + # The `+deprecated` is added to distinguish content type + # as defined by `API::API` vs ex. `API::Repositories` + content_type :txt, 'text/plain+deprecated' + + before do + # the feature flag workaround is only for `.txt` + api_format = env[Grape::Env::API_FORMAT] + next unless api_format == :txt + + # get all defined content-types for the endpoint + api_endpoint = env[Grape::Env::API_ENDPOINT] + content_types = api_endpoint&.namespace_stackable_with_hash(:content_types).to_h + + # Only overwrite `text/plain+deprecated` + if content_types[api_format] == 'text/plain+deprecated' + if Feature.enabled?(:api_always_use_application_json) + content_type 'application/json' + else + content_type 'text/plain' + end + end end # Ensure the namespace is right, otherwise we might load Grape::API::Helpers @@ -249,6 +268,7 @@ module API mount ::API::Release::Links mount ::API::RemoteMirrors mount ::API::Repositories + mount ::API::ResourceAccessTokens mount ::API::Search mount ::API::Services mount ::API::Settings @@ -294,4 +314,4 @@ module API end end -API::API.prepend_if_ee('::EE::API::API') +API::API.prepend_ee_mod diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 8b14e16b495..b883f83cc19 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -8,15 +8,6 @@ module API feature_category :authentication_and_authorization resource :applications do - helpers do - def validate_redirect_uri(value) - uri = ::URI.parse(value) - !uri.is_a?(URI::HTTP) || uri.host - rescue URI::InvalidURIError - false - end - end - desc 'Create a new application' do detail 'This feature was introduced in GitLab 10.5' success Entities::ApplicationWithSecret @@ -30,13 +21,6 @@ module API desc: 'Application will be used where the client secret is confidential' end post do - # Validate that host in uri is specified - # Please remove it when https://github.com/doorkeeper-gem/doorkeeper/pull/1440 is merged - # and the doorkeeper gem version is bumped - unless validate_redirect_uri(declared_params[:redirect_uri]) - render_api_error!({ redirect_uri: ["must be an absolute URI."] }, :bad_request) - end - application = Doorkeeper::Application.new(declared_params) if application.save diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index 6c8b3a1ba4a..1796d51324f 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -72,6 +72,7 @@ module API namespace 'users' do format :txt + content_type :txt, 'text/plain' desc 'Authenticate user against conan CLI' do detail 'This feature was introduced in GitLab 12.2' diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index e3cacc4132f..f138f400601 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -8,12 +8,14 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do + require_packages_enabled! + not_found! unless ::Feature.enabled?(:debian_packages, user_group) authorize_read_package!(user_group) end - namespace ':id/-/packages/debian' do + namespace ':id/packages/debian' do include DebianPackageEndpoints end end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb index c95c75b7e5c..e7689b3feff 100644 --- a/lib/api/debian_package_endpoints.rb +++ b/lib/api/debian_package_endpoints.rb @@ -32,6 +32,7 @@ module API helpers ::API::Helpers::Packages::BasicAuthHelpers format :txt + content_type :txt, 'text/plain' rescue_from ArgumentError do |e| render_api_error!(e.message, 400) @@ -50,33 +51,33 @@ module API end namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg desc 'The Release file signature' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Release.gpg' do not_found! end - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release desc 'The unsigned Release file' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Release' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Release' end - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease + # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease desc 'The signed Release file' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'InRelease' do not_found! end @@ -87,12 +88,12 @@ module API end namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages desc 'The binary files index' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Packages' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Packages' @@ -107,7 +108,7 @@ module API end namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name + # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name params do requires :file_name, type: String, desc: 'The Debian File Name' end @@ -115,7 +116,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get ':file_name', requirements: FILE_NAME_REQUIREMENTS do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO File' diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index f8129c18dff..8c0db42a448 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -8,27 +8,29 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do + require_packages_enabled! + not_found! unless ::Feature.enabled?(:debian_packages, user_project) authorize_read_package! end - namespace ':id/-/packages/debian' do + namespace ':id/packages/debian' do include DebianPackageEndpoints params do requires :file_name, type: String, desc: 'The file name' end - namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name + # PUT {projects|groups}/:id/packages/debian/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put do authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) @@ -42,8 +44,8 @@ module API forbidden! end - # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + # PUT {projects|groups}/:id/packages/debian/:file_name/authorize + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put 'authorize' do authorize_workhorse!( subject: authorized_user_project, diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 5fab590eb4e..30ec4e52b2a 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -28,8 +28,6 @@ module API use :pagination end get 'deploy_tokens' do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, default_enabled: true) - authenticated_as_admin! present paginate(DeployToken.all), with: Entities::DeployToken @@ -39,10 +37,6 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_project, default_enabled: true) - end - params do use :pagination end @@ -102,10 +96,6 @@ module API requires :id, type: String, desc: 'The ID of a group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_group, default_enabled: true) - end - params do use :pagination end diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index e9572a8d430..2468c1f9b18 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -31,6 +31,7 @@ module API expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services + expose :asset_proxy_allowlist, as: :asset_proxy_whitelist end end end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 4aa5c9b7236..b392e7831e5 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -6,6 +6,7 @@ module API include UsersHelper expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } expose :bio, :bio_html, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title + expose :bot?, as: :bot expose :work_information do |user| work_information(user) end diff --git a/lib/api/events.rb b/lib/api/events.rb index 233c62b5389..db5ed7b7e6e 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -6,7 +6,7 @@ module API include APIGuard helpers ::API::Helpers::EventsHelpers - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } feature_category :users diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index 2d978019f2a..ea30f17522e 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -30,20 +30,6 @@ module API str.gsub(/![[:alpha:]]/) { |s| s[1..].upcase } end - def find_project!(id) - # based on API::Helpers::Packages::BasicAuthHelpers#authorized_project_find! - - project = find_project(id) - - return project if project && can?(current_user, :read_project, project) - - if current_user - not_found!('Project') - else - unauthorized! - end - end - def find_module not_found! unless Feature.enabled?(:go_proxy, user_project) @@ -74,7 +60,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :module_name, type: String, desc: 'Module name', coerce_with: ->(val) { CGI.unescape(val) } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, authenticate_non_public: true resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do authorize_read_package! diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 7fbf4445116..bea538441ee 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :groups, requirements: ::API::Labels::LABEL_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the group' do detail 'This feature was added in GitLab 11.8' success Entities::GroupLabel diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 79af9c37378..0abb21c9831 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -119,11 +119,10 @@ module API def find_project!(id) project = find_project(id) - if can?(current_user, :read_project, project) - project - else - not_found!('Project') - end + return project if can?(current_user, :read_project, project) + return unauthorized! if authenticate_non_public? + + not_found!('Project') end # rubocop: disable CodeReuse/ActiveRecord @@ -139,11 +138,10 @@ module API def find_group!(id) group = find_group(id) - if can?(current_user, :read_group, group) - group - else - not_found!('Group') - end + return group if can?(current_user, :read_group, group) + return unauthorized! if authenticate_non_public? + + not_found!('Group') end def check_namespace_access(namespace) @@ -657,6 +655,10 @@ module API Gitlab::Shell.secret_token end + def authenticate_non_public? + route_authentication_setting[:authenticate_non_public] && !current_user + end + def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 6798c4d284b..71a18524104 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -138,7 +138,7 @@ module API parent = noteable_parent(noteable) ::Discussions::ResolveService.new(parent, current_user, one_or_more_discussions: discussion).execute else - discussion.unresolve! + ::Discussions::UnresolveService.new(discussion, current_user).execute end present discussion, with: Entities::Discussion diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 12bb6e77c3e..6de80c17960 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -52,7 +52,9 @@ module API actor.update_last_used_at! check_result = begin - access_check!(actor, params) + Gitlab::Auth::CurrentUserMode.bypass_session!(actor.user&.id) do + access_check!(actor, params) + end rescue Gitlab::GitAccess::ForbiddenError => e # The return code needs to be 401. If we return 403 # the custom message we return won't be shown to the user diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 73723a96401..87ad79d601f 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -52,6 +52,8 @@ module API def check_agent_token forbidden! unless agent_token + + forbidden! unless Gitlab::Kas.included_in_gitlab_com_rollout?(agent.project) end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index e14a4a5e680..c09b01f5b4e 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -82,7 +82,8 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace.raw + # The trace can be nil bu body method expects a string as an argument. + trace = build.trace.raw || '' body trace end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c9f29865664..aa3746dae42 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -9,10 +9,14 @@ module API feature_category :issue_tracking + LABEL_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + name: API::NO_SLASH_URL_PART_REGEX, + label_id: API::NO_SLASH_URL_PART_REGEX) + params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: LABEL_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::ProjectLabel end diff --git a/lib/api/members.rb b/lib/api/members.rb index 9bea74e2ce9..42f608102b3 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -137,12 +137,14 @@ module API authorize_admin_source!(source_type, source) member = source_members(source).find_by!(user_id: params[:user_id]) - updated_member = - ::Members::UpdateService - .new(current_user, declared_params(include_missing: false)) - .execute(member) - if updated_member.valid? + result = ::Members::UpdateService + .new(current_user, declared_params(include_missing: false)) + .execute(member) + + updated_member = result[:member] + + if result[:status] == :success present_members updated_member else render_validation_error!(updated_member) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index ab0e9b95e4a..cff0866c65e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -26,6 +26,7 @@ module API %i[ assignee_id assignee_ids + reviewer_ids description labels add_labels @@ -160,7 +161,8 @@ module API helpers do params :optional_params do optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' - optional :assignee_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The array of user IDs to assign issue' + optional :assignee_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of assignee ids' + optional :reviewer_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of reviewer ids' optional :description, type: String, desc: 'The description of the merge request' optional :labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' optional :add_labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' @@ -359,7 +361,7 @@ module API with: Entities::MergeRequestChanges, current_user: current_user, project: user_project, - access_raw_diffs: params.fetch(:access_raw_diffs, false) + access_raw_diffs: to_boolean(params.fetch(:access_raw_diffs, false)) end desc 'Get the merge request pipelines' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2d09ad01757..fca68c3606b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -295,6 +295,8 @@ module API optional :namespace_path, type: String, desc: 'The path of the namespace that the project will be forked into' optional :path, type: String, desc: 'The path that will be assigned to the fork' optional :name, type: String, desc: 'The name that will be assigned to the fork' + optional :description, type: String, desc: 'The description that will be assigned to the fork' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' end post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 8af8ffc3b63..353f2ed1c25 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -170,6 +170,67 @@ module API not_found!("Merge Base") end end + + desc 'Generates a changelog section for a release' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + requires :version, + type: String, + regexp: Gitlab::Regex.unbounded_semver_regex, + desc: 'The version of the release, using the semantic versioning format' + + requires :from, + type: String, + desc: 'The first commit in the range of commits to use for the changelog' + + requires :to, + type: String, + desc: 'The last commit in the range of commits to use for the changelog' + + optional :date, + type: DateTime, + desc: 'The date and time of the release' + + optional :branch, + type: String, + desc: 'The branch to commit the changelog changes to' + + optional :trailer, + type: String, + desc: 'The Git trailer to use for determining if commits are to be included in the changelog', + default: ::Repositories::ChangelogService::DEFAULT_TRAILER + + optional :file, + type: String, + desc: 'The file to commit the changelog changes to', + default: ::Repositories::ChangelogService::DEFAULT_FILE + + optional :message, + type: String, + desc: 'The commit message to use when committing the changelog' + end + post ':id/repository/changelog' do + not_found! unless Feature.enabled?(:changelog_api, user_project) + + branch = params[:branch] || user_project.default_branch_or_master + access = Gitlab::UserAccess.new(current_user, container: user_project) + + unless access.can_push_to_branch?(branch) + forbidden!("You are not allowed to commit a changelog on this branch") + end + + service = ::Repositories::ChangelogService.new( + user_project, + current_user, + **declared_params(include_missing: false) + ) + + service.execute + status(200) + rescue => ex + render_api_error!("Failed to generate the changelog: #{ex.message}", 500) + end end end end diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb new file mode 100644 index 00000000000..66948f9eaf3 --- /dev/null +++ b/lib/api/resource_access_tokens.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module API + class ResourceAccessTokens < ::API::Base + include PaginationParams + + before { authenticate! } + + feature_category :authentication_and_authorization + + %w[project].each do |source_type| + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get list of all access tokens for the specified resource' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + get ":id/access_tokens" do + resource = find_source(source_type, params[:id]) + + next unauthorized! unless has_permission_to_read?(resource) + + tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute + + present paginate(tokens), with: Entities::PersonalAccessToken + end + + desc 'Revoke a resource access token' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :token_id, type: String, desc: "The ID of the token" + end + delete ':id/access_tokens/:token_id' do + resource = find_source(source_type, params[:id]) + token = find_token(resource, params[:token_id]) + + if token.nil? + next not_found!("Could not find #{source_type} access token with token_id: #{params[:token_id]}") + end + + service = ::ResourceAccessTokens::RevokeService.new( + current_user, + resource, + token + ).execute + + service.success? ? no_content! : bad_request!(service.message) + end + + desc 'Create a resource access token' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :name, type: String, desc: "Resource access token name" + requires :scopes, type: Array[String], desc: "The permissions of the token" + optional :expires_at, type: Date, desc: "The expiration date of the token" + end + post ':id/access_tokens' do + resource = find_source(source_type, params[:id]) + + token_response = ::ResourceAccessTokens::CreateService.new( + current_user, + resource, + declared_params + ).execute + + if token_response.success? + present token_response.payload[:access_token], with: Entities::PersonalAccessToken + else + bad_request!(token_response.message) + end + end + end + end + + helpers do + def find_source(source_type, id) + public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend + end + + def find_token(resource, token_id) + PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).find_by_id(token_id) + end + + def has_permission_to_read?(resource) + can?(current_user, :project_bot_access, resource) || can?(current_user, :admin_resource_access_tokens, resource) + end + end + end +end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index f329a94adf2..6b1ad33d84b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -42,7 +42,8 @@ module API optional :asset_proxy_enabled, type: Boolean, desc: 'Enable proxying of assets' optional :asset_proxy_url, type: String, desc: 'URL of the asset proxy server' optional :asset_proxy_secret_key, type: String, desc: 'Shared secret with the asset proxy server' - optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' + optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :asset_proxy_allowlist instead. Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' + optional :asset_proxy_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_ci_config_path, type: String, desc: 'The instance default CI configuration path for new projects' @@ -211,6 +212,11 @@ module API attrs[:abuse_notification_email] = attrs.delete(:admin_notification_email) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:asset_proxy_whitelist) + attrs[:asset_proxy_allowlist] = attrs.delete(:asset_proxy_whitelist) + end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) diff --git a/lib/api/snippet_repository_storage_moves.rb b/lib/api/snippet_repository_storage_moves.rb index 1a5b41eb1ec..84dbc03ba33 100644 --- a/lib/api/snippet_repository_storage_moves.rb +++ b/lib/api/snippet_repository_storage_moves.rb @@ -58,9 +58,14 @@ module API resource :snippets do helpers do def user_snippet - Snippet.find_by(id: params[:id]) # rubocop: disable CodeReuse/ActiveRecord + @user_snippet ||= Snippet.find_by(id: params[:id]) # rubocop: disable CodeReuse/ActiveRecord end end + + before do + not_found!('Snippet') unless user_snippet + end + desc 'Get a list of all snippets repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' success Entities::SnippetRepositoryStorageMove diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 914bab52929..87dc1358a51 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -6,6 +6,9 @@ module API before { authenticate! } + SUBSCRIBE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + subscribable_id: API::NO_SLASH_URL_PART_REGEX) + subscribables = [ { type: 'merge_requests', @@ -44,7 +47,7 @@ module API requires :id, type: String, desc: "The #{source_type} ID" requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource source_type.pluralize, requirements: SUBSCRIBE_ENDPOINT_REQUIREMENTS do desc 'Subscribe to a resource' do success subscribable[:entity] end diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index a024d6de874..7921700e365 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -12,12 +12,13 @@ module API end params do requires :id, type: String, desc: 'The suggestion ID' + optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put ':id/apply' do suggestion = Suggestion.find_by_id(params[:id]) if suggestion - apply_suggestions(suggestion, current_user) + apply_suggestions(suggestion, current_user, params[:commit_message]) else render_api_error!(_('Suggestion is not applicable as the suggestion was not found.'), :not_found) end @@ -28,6 +29,7 @@ module API end params do requires :ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: "An array of suggestion ID's" + optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put 'batch_apply' do ids = params[:ids] @@ -35,7 +37,7 @@ module API suggestions = Suggestion.id_in(ids) if suggestions.size == ids.length - apply_suggestions(suggestions, current_user) + apply_suggestions(suggestions, current_user, params[:commit_message]) else render_api_error!(_('Suggestions are not applicable as one or more suggestions were not found.'), :not_found) end @@ -43,10 +45,10 @@ module API end helpers do - def apply_suggestions(suggestions, current_user) + def apply_suggestions(suggestions, current_user, message) authorize_suggestions(*suggestions) - result = ::Suggestions::ApplyService.new(current_user, *suggestions).execute + result = ::Suggestions::ApplyService.new(current_user, *suggestions, message: message).execute if result[:status] == :success present suggestions, with: Entities::Suggestion, current_user: current_user diff --git a/lib/api/users.rb b/lib/api/users.rb index cee09f60a2b..36368adf2f0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,7 +6,7 @@ module API include APIGuard include Helpers::CustomAttributes - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] diff --git a/lib/api/version.rb b/lib/api/version.rb index f8072658cc6..86eb34ca589 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -5,7 +5,7 @@ module API helpers ::API::Helpers::GraphqlHelpers include APIGuard - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } before { authenticate! } diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index c67fe24d456..6f87b7b7d3c 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -4,7 +4,7 @@ module Atlassian module JiraConnect class Client < Gitlab::HTTP def self.generate_update_sequence_id - Gitlab::Metrics::System.monotonic_time.to_i + (Time.now.utc.to_f * 1000).round end def initialize(base_uri, shared_secret) @@ -33,8 +33,6 @@ module Atlassian private def store_ff_info(project:, feature_flags:, **opts) - return unless Feature.enabled?(:jira_sync_feature_flags, project) - items = feature_flags.map { |flag| ::Atlassian::JiraConnect::Serializers::FeatureFlagEntity.represent(flag, opts) } items.reject! { |item| item.issue_keys.empty? } @@ -57,8 +55,6 @@ module Atlassian end def store_deploy_info(project:, deployments:, **opts) - return unless Feature.enabled?(:jira_sync_deployments, project) - items = deployments.map { |d| ::Atlassian::JiraConnect::Serializers::DeploymentEntity.represent(d, opts) } items.reject! { |d| d.issue_keys.empty? } @@ -69,8 +65,6 @@ module Atlassian end def store_build_info(project:, pipelines:, update_sequence_id: nil) - return unless Feature.enabled?(:jira_sync_builds, project) - builds = pipelines.map do |pipeline| build = ::Atlassian::JiraConnect::Serializers::BuildEntity.represent( pipeline, diff --git a/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb index e17c150aacb..3193d5bbd1e 100644 --- a/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb +++ b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb @@ -50,7 +50,7 @@ module Atlassian # edit path as an interim solution. def summary(strategies = flag.strategies) { - url: project_url(flag.project) + "/-/feature_flags/#{flag.id}/edit", + url: edit_project_feature_flag_url(flag.project, flag), lastUpdated: flag.updated_at.iso8601, status: { enabled: flag.active, diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 0f6ed847dea..42cfff98239 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -137,7 +137,7 @@ module Backup if s == DEFAULT_EXCLUDE '--exclude=' + s elsif fmt == :rsync - '--exclude=/' + s + '--exclude=/' + File.join(File.basename(app_files_dir), s) elsif fmt == :tar '--exclude=./' + s end diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 55dc426edaf..4c14ee7299b 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -59,7 +59,9 @@ module Banzai end def self.determine_allowlist(application_settings) - application_settings.asset_proxy_whitelist.presence || [Gitlab.config.gitlab.host] + application_settings.try(:asset_proxy_allowlist).presence || + application_settings.try(:asset_proxy_whitelist).presence || + [Gitlab.config.gitlab.host] end end end diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb new file mode 100644 index 00000000000..ad32e9afbf5 --- /dev/null +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class MarkdownPostEscapeFilter < HTML::Pipeline::Filter + LITERAL_KEYWORD = MarkdownPreEscapeFilter::LITERAL_KEYWORD + LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}}.freeze + NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze + SPAN_REGEX = %r{<span>(.*?)</span>}.freeze + + def call + return doc unless result[:escaped_literals] + + # For any literals that actually didn't get escape processed + # (for example in code blocks), remove the special sequence. + html.gsub!(NOT_LITERAL_REGEX, '\1') + + # Replace any left over literal sequences with `span` so that our + # reference processing is short-circuited + html.gsub!(LITERAL_REGEX, '<span>\1</span>') + + # Since literals are converted in links, we need to remove any surrounding `span`. + # Note: this could have been done in the renderer, + # Banzai::Renderer::CommonMark::HTML. However, we eventually want to use + # the built-in compiled renderer, rather than the ruby version, for speed. + # So let's do this work here. + doc.css('a').each do |node| + node.attributes['href'].value = node.attributes['href'].value.gsub(SPAN_REGEX, '\1') if node.attributes['href'] + node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title'] + end + + doc.css('code').each do |node| + node.attributes['lang'].value = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang'] + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb new file mode 100644 index 00000000000..9fd77c48659 --- /dev/null +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # In order to allow a user to short-circuit our reference shortcuts + # (such as # or !), the user should be able to escape them, like \#. + # CommonMark supports this, however it removes all information about + # what was actually a literal. In order to short-circuit the reference, + # we must surround backslash escaped ASCII punctuation with a custom sequence. + # This way CommonMark will properly handle the backslash escaped chars + # but we will maintain knowledge (the sequence) that it was a literal. + # + # We need to surround the character, not just prefix it. It could + # get converted into an entity by CommonMark and we wouldn't know how many + # characters there are. The entire literal needs to be surrounded with + # a `span` tag, which short-circuits our reference processing. + # + # We can't use a custom HTML tag since we could be initially surrounding + # text in an href, and then CommonMark will not be able to parse links + # properly. So we use `cmliteral-` and `-cmliteral` + # + # https://spec.commonmark.org/0.29/#backslash-escapes + # + # This filter does the initial surrounding, and MarkdownPostEscapeFilter + # does the conversion into span tags. + class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter + ASCII_PUNCTUATION = %r{([\\][!"#$%&'()*+,-./:;<=>?@\[\\\]^_`{|}~])}.freeze + LITERAL_KEYWORD = 'cmliteral' + + def call + return @text unless Feature.enabled?(:honor_escaped_markdown, context[:group] || context[:project]&.group) + + @text.gsub(ASCII_PUNCTUATION) do |match| + # The majority of markdown does not have literals. If none + # are found, we can bypass the post filter + result[:escaped_literals] = true + + "#{LITERAL_KEYWORD}-#{match}-#{LITERAL_KEYWORD}" + end + end + end + end +end diff --git a/lib/banzai/filter/truncate_source_filter.rb b/lib/banzai/filter/truncate_source_filter.rb index c903b83d868..44f88b253d9 100644 --- a/lib/banzai/filter/truncate_source_filter.rb +++ b/lib/banzai/filter/truncate_source_filter.rb @@ -6,7 +6,9 @@ module Banzai def call return text unless context.key?(:limit) - text.truncate_bytes(context[:limit]) + # Use three dots instead of the ellipsis Unicode character because + # some clients show the raw Unicode value in the merge commit. + text.truncate_bytes(context[:limit], omission: '...') end end end diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index b64f13cde47..1da0f72996b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -5,7 +5,9 @@ module Banzai class PlainMarkdownPipeline < BasePipeline def self.filters FilterArray[ - Filter::MarkdownFilter + Filter::MarkdownPreEscapeFilter, + Filter::MarkdownFilter, + Filter::MarkdownPostEscapeFilter ] end end diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index af274ee1299..9e53b525fe3 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -4,17 +4,22 @@ module BulkImports module Common module Extractors class GraphqlExtractor - def initialize(query) - @query = query[:query] + def initialize(options = {}) + @query = options[:query] end def extract(context) client = graphql_client(context) - client.execute( + response = client.execute( client.parse(query.to_s), query.variables(context.entity) ).original_hash.deep_dup + + BulkImports::Pipeline::ExtractedData.new( + data: response.dig(*query.data_path), + page_info: response.dig(*query.page_info_path) + ) end private @@ -27,10 +32,6 @@ module BulkImports token: context.configuration.access_token ) end - - def parsed_query - @parsed_query ||= graphql_client.parse(query.to_s) - end end end end diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb index 4540b892c88..8644f3c9dcb 100644 --- a/lib/bulk_imports/common/loaders/entity_loader.rb +++ b/lib/bulk_imports/common/loaders/entity_loader.rb @@ -7,7 +7,7 @@ module BulkImports def initialize(*args); end def load(context, entity) - context.entity.bulk_import.entities.create!(entity) + context.bulk_import.entities.create!(entity) end end end diff --git a/lib/bulk_imports/common/transformers/hash_key_digger.rb b/lib/bulk_imports/common/transformers/hash_key_digger.rb deleted file mode 100644 index b4897b5b2bf..00000000000 --- a/lib/bulk_imports/common/transformers/hash_key_digger.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Transformers - class HashKeyDigger - def initialize(options = {}) - @key_path = options[:key_path] - end - - def transform(_, data) - raise ArgumentError, "Given data must be a Hash" unless data.is_a?(Hash) - - data.dig(*Array.wrap(key_path)) - end - - private - - attr_reader :key_path - end - end - end -end diff --git a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb deleted file mode 100644 index b32ab28fdbb..00000000000 --- a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Transformers - class UnderscorifyKeysTransformer - def initialize(options = {}) - @options = options - end - - def transform(_, data) - data.deep_transform_keys do |key| - key.to_s.underscore - end - end - end - end - end -end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index 5c5e686cec5..b01fb6f68ac 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -9,9 +9,11 @@ module BulkImports def extract(context) encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) - http_client(context.entity.bulk_import.configuration) + response = http_client(context.configuration) .each_page(:get, "groups/#{encoded_parent_path}/subgroups") .flat_map(&:itself) + + BulkImports::Pipeline::ExtractedData.new(data: response) end private diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index 2bc0f60baa2..169c61247c3 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -12,18 +12,18 @@ module BulkImports group(fullPath: $full_path) { name path - fullPath + full_path: fullPath description visibility - emailsDisabled - lfsEnabled - mentionsDisabled - projectCreationLevel - requestAccessEnabled - requireTwoFactorAuthentication - shareWithGroupLock - subgroupCreationLevel - twoFactorGracePeriod + emails_disabled: emailsDisabled + lfs_enabled: lfsEnabled + mentions_disabled: mentionsDisabled + project_creation_level: projectCreationLevel + request_access_enabled: requestAccessEnabled + require_two_factor_authentication: requireTwoFactorAuthentication + share_with_group_lock: shareWithGroupLock + subgroup_creation_level: subgroupCreationLevel + two_factor_grace_period: twoFactorGracePeriod } } GRAPHQL @@ -32,6 +32,18 @@ module BulkImports def variables(entity) { full_path: entity.source_full_path } end + + def base_path + %w[data group] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end end end end diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb new file mode 100644 index 00000000000..cd57b46b9f4 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetLabelsQuery + extend self + + def to_s + <<-'GRAPHQL' + query ($full_path: ID!, $cursor: String) { + group(fullPath: $full_path) { + labels(first: 100, after: $cursor) { + page_info: pageInfo { + end_cursor: endCursor + has_next_page: hasNextPage + } + nodes { + title + description + color + } + } + } + } + GRAPHQL + end + + def variables(entity) + { + full_path: entity.source_full_path, + cursor: entity.next_page_for(:labels) + } + end + + def base_path + %w[data group labels] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/labels_loader.rb b/lib/bulk_imports/groups/loaders/labels_loader.rb new file mode 100644 index 00000000000..b8c9ba9609c --- /dev/null +++ b/lib/bulk_imports/groups/loaders/labels_loader.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Loaders + class LabelsLoader + def initialize(*); end + + def load(context, data) + Labels::CreateService.new(data).execute(group: context.group) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb index 5169e292180..8c6f089e8a4 100644 --- a/lib/bulk_imports/groups/pipelines/group_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -10,8 +10,6 @@ module BulkImports extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery - transformer Common::Transformers::HashKeyDigger, key_path: %w[data group] - transformer Common::Transformers::UnderscorifyKeysTransformer transformer Common::Transformers::ProhibitedAttributesTransformer transformer Groups::Transformers::GroupAttributesTransformer diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb new file mode 100644 index 00000000000..6b48106e5b8 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class LabelsPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::GraphqlExtractor, + query: BulkImports::Groups::Graphql::GetLabelsQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + loader BulkImports::Groups::Loaders::LabelsLoader + + def after_run(context, extracted_data) + context.entity.update_tracker_for( + relation: :labels, + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + + if extracted_data.has_next_page? + run(context) + end + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index 6e1b86e9515..c01a4ec025d 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -8,14 +8,7 @@ module BulkImports end def execute - bulk_import = entity.bulk_import - configuration = bulk_import.configuration - - context = BulkImports::Pipeline::Context.new( - current_user: bulk_import.user, - entity: entity, - configuration: configuration - ) + context = BulkImports::Pipeline::Context.new(entity) pipelines.each { |pipeline| pipeline.new.run(context) } @@ -29,7 +22,8 @@ module BulkImports def pipelines [ BulkImports::Groups::Pipelines::GroupPipeline, - BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline + BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + BulkImports::Groups::Pipelines::LabelsPipeline ] end end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 06b81b5da14..3d2a74cd4b3 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -22,10 +22,6 @@ module BulkImports @loaders ||= instantiate(self.class.get_loader) end - def after_run - @after_run ||= self.class.after_run_callback - end - def pipeline @pipeline ||= self.class.name end @@ -52,10 +48,6 @@ module BulkImports class_attributes[:loader] = { klass: klass, options: options } end - def after_run(&block) - class_attributes[:after_run] = block - end - def get_extractor class_attributes[:extractor] end @@ -68,10 +60,6 @@ module BulkImports class_attributes[:loader] end - def after_run_callback - class_attributes[:after_run] - end - def abort_on_failure! class_attributes[:abort_on_failure] = true end diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index ad19f5cad7d..cda03c04ef2 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -3,30 +3,23 @@ module BulkImports module Pipeline class Context - include Gitlab::Utils::LazyAttributes + attr_reader :entity, :bulk_import - Attribute = Struct.new(:name, :type) - - PIPELINE_ATTRIBUTES = [ - Attribute.new(:current_user, User), - Attribute.new(:entity, ::BulkImports::Entity), - Attribute.new(:configuration, ::BulkImports::Configuration) - ].freeze - - def initialize(args) - assign_attributes(args) + def initialize(entity) + @entity = entity + @bulk_import = entity.bulk_import end - private + def group + entity.group + end - PIPELINE_ATTRIBUTES.each do |attr| - lazy_attr_reader attr.name, type: attr.type + def current_user + bulk_import.user end - def assign_attributes(values) - values.slice(*PIPELINE_ATTRIBUTES.map(&:name)).each do |name, value| - instance_variable_set("@#{name}", value) - end + def configuration + bulk_import.configuration end end end diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb new file mode 100644 index 00000000000..685a91a4afe --- /dev/null +++ b/lib/bulk_imports/pipeline/extracted_data.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + class ExtractedData + attr_reader :data + + def initialize(data: nil, page_info: {}) + @data = Array.wrap(data) + @page_info = page_info + end + + def has_next_page? + @page_info['has_next_page'] + end + + def next_page + @page_info['end_cursor'] + end + + def each(&block) + data.each(&block) + end + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 11fb9722173..1c4ee154874 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -10,9 +10,11 @@ module BulkImports def run(context) raise MarkedAsFailedError if marked_as_failed?(context) - info(context, message: 'Pipeline started', pipeline_class: pipeline) + info(context, message: 'Pipeline started') - Array.wrap(extracted_data_from(context)).each do |entry| + extracted_data = extracted_data_from(context) + + extracted_data&.each do |entry| transformers.each do |transformer| entry = run_pipeline_step(:transformer, transformer.class.name, context) do transformer.transform(context, entry) @@ -24,25 +26,29 @@ module BulkImports end end - after_run.call(context) if after_run.present? + after_run(context, extracted_data) if respond_to?(:after_run) + + info(context, message: 'Pipeline finished') rescue MarkedAsFailedError log_skip(context) end private # rubocop:disable Lint/UselessAccessModifier - def run_pipeline_step(type, class_name, context) + def run_pipeline_step(step, class_name, context) raise MarkedAsFailedError if marked_as_failed?(context) - info(context, type => class_name) + info(context, pipeline_step: step, step_class: class_name) yield rescue MarkedAsFailedError - log_skip(context, type => class_name) + log_skip(context, step => class_name) rescue => e - log_import_failure(e, context) + log_import_failure(e, step, context) mark_as_failed(context) if abort_on_failure? + + nil end def extracted_data_from(context) @@ -72,10 +78,11 @@ module BulkImports info(context, log) end - def log_import_failure(exception, context) + def log_import_failure(exception, step, context) attributes = { bulk_import_entity_id: context.entity.id, pipeline_class: pipeline, + pipeline_step: step, exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id @@ -95,7 +102,8 @@ module BulkImports def log_base_params(context) { bulk_import_entity_id: context.entity.id, - bulk_import_entity_type: context.entity.source_type + bulk_import_entity_type: context.entity.source_type, + pipeline_class: pipeline } end diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb new file mode 100644 index 00000000000..20e0cb333dc --- /dev/null +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + class UsageMetricDefinitionGenerator < Rails::Generators::Base + Directory = Struct.new(:name, :time_frame) do + def match?(str) + (name == str || time_frame == str) && str != 'none' + end + end + + TIME_FRAME_DIRS = [ + Directory.new('counts_7d', '7d'), + Directory.new('counts_28d', '28d'), + Directory.new('counts_all', 'all'), + Directory.new('settings', 'none'), + Directory.new('license', 'none') + ].freeze + + VALID_INPUT_DIRS = (TIME_FRAME_DIRS.flat_map { |d| [d.name, d.time_frame] } - %w(none)).freeze + + source_root File.expand_path('../../../generator_templates/usage_metric_definition', __dir__) + + desc 'Generates a metric definition yml file' + + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' + class_option :dir, + type: :string, desc: "Indicates the metric location. It must be one of: #{VALID_INPUT_DIRS.join(', ')}" + + argument :key_path, type: :string, desc: 'Unique JSON key path for the metric' + + def create_metric_file + validate! + + template "metric_definition.yml", file_path + end + + def time_frame + directory&.time_frame + end + + def distribution + value = ['ce'] + value << 'ee' if ee? + value + end + + private + + def file_path + path = File.join('config', 'metrics', directory&.name, "#{file_name}.yml") + path = File.join('ee', path) if ee? + path + end + + def validate! + raise "--dir option is required" unless input_dir.present? + raise "Invalid dir #{input_dir}, allowed options are #{VALID_INPUT_DIRS.join(', ')}" unless directory.present? + end + + def ee? + options[:ee] + end + + def input_dir + options[:dir] + end + + def file_name + key_path.split('.').last + end + + def directory + @directory ||= TIME_FRAME_DIRS.find { |d| d.match?(input_dir) } + end + end +end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 0f2fd01e3c7..d84196b0bc2 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -48,6 +48,10 @@ module Gitlab Gitlab.config.gitlab.url == COM_URL || gl_subdomain? end + def self.com + yield if com? + end + def self.staging? Gitlab.config.gitlab.url == STAGING_COM_URL end @@ -118,6 +122,7 @@ module Gitlab def self.maintenance_mode? return false unless ::Feature.enabled?(:maintenance_mode) + return false unless ::Gitlab::CurrentSettings.current_application_settings? ::Gitlab::CurrentSettings.maintenance_mode end diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index ce09ffd87ee..d3ce5cc8c74 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -47,3 +47,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload.prepend_if_ee('EE::Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb new file mode 100644 index 00000000000..f85b2248aeb --- /dev/null +++ b/lib/gitlab/auth/u2f_webauthn_converter.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + class U2fWebauthnConverter + def initialize(u2f_registration) + @u2f_registration = u2f_registration + end + + def convert + now = Time.current + + converted_credential = WebAuthn::U2fMigrator.new( + app_id: Gitlab.config.gitlab.url, + certificate: u2f_registration.certificate, + key_handle: u2f_registration.key_handle, + public_key: u2f_registration.public_key, + counter: u2f_registration.counter + ).credential + + { + credential_xid: Base64.strict_encode64(converted_credential.id), + public_key: Base64.strict_encode64(converted_credential.public_key), + counter: u2f_registration.counter || 0, + name: u2f_registration.name || '', + user_id: u2f_registration.user_id, + u2f_registration_id: u2f_registration.id, + created_at: now, + updated_at: now + } + end + + private + + attr_reader :u2f_registration + end + end +end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d1b9062a23c..9f4d6557023 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -33,7 +33,7 @@ module Gitlab next unless job.queue == self.queue next unless migration_class == steal_class - next if block_given? && !(yield migration_args) + next if block_given? && !(yield job) begin perform(migration_class, migration_args) if job.delete diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb index b8c14aa2573..091e6660bac 100644 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -16,26 +16,9 @@ module Gitlab def perform(start_id, end_id) old_registrations = U2fRegistration.where(id: start_id..end_id) old_registrations.each_slice(100) do |slice| - now = Time.now values = slice.map do |u2f_registration| - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter || 0, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: now, - updated_at: now - } + converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration) + converter.convert end WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb new file mode 100644 index 00000000000..3d3970f50e1 --- /dev/null +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateUuidsForSecurityFindings + NOP_RELATION = Class.new { def each_batch(*); end } + + def self.security_findings + NOP_RELATION.new + end + + def perform(_scan_ids); end + end + end +end + +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..ca61118a06c --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings + DELETE_BATCH_SIZE = 100 + + # rubocop:disable Gitlab/NamespacedClass + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + end + # rubocop:enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + batch = VulnerabilitiesFinding.where(id: start_id..end_id) + + cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id)) + + query = VulnerabilitiesFinding + .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids') + .distinct + .with(cte.to_arel) + .from(cte.alias_to(Arel.sql('batch'))) + .joins( + %( + INNER JOIN + vulnerability_occurrences ON + vulnerability_occurrences.report_type = batch.report_type AND + vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND + vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND + vulnerability_occurrences.project_id = batch.project_id + )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id') + .having('COUNT(*) > 1') + + ids_to_delete = [] + + query.to_a.each do |record| + # We want to keep the latest finding since it might have recent metadata + duplicate_ids = record.ids.uniq.sort + duplicate_ids.pop + ids_to_delete.concat(duplicate_ids) + + if ids_to_delete.size == DELETE_BATCH_SIZE + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all + ids_to_delete.clear + end + end + + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all if ids_to_delete.any? + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 6d7b9a86e69..8fa0db5fd4b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -6,6 +6,7 @@ module Gitlab module Models # isolated Namespace model class Namespace < ApplicationRecord + include FeatureGate include ::Gitlab::VisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb new file mode 100644 index 00000000000..617017faa58 --- /dev/null +++ b/lib/gitlab/changelog/committer.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class used for committing a release's changelog to a Git repository. + class Committer + CommitError = Class.new(StandardError) + + def initialize(project, user) + @project = project + @user = user + end + + # Commits a release's changelog to a file on a branch. + # + # The `release` argument is a `Gitlab::Changelog::Release` for which to + # update the changelog. + # + # The `file` argument specifies the path to commit the changes to. + # + # The `branch` argument specifies the branch to commit the changes on. + # + # The `message` argument specifies the commit message to use. + def commit(release:, file:, branch:, message:) + # When retrying, we need to reprocess the existing changelog from + # scratch, otherwise we may end up throwing away changes. As such, all + # the logic is contained within the retry block. + Retriable.retriable(on: CommitError) do + commit = Gitlab::Git::Commit.last_for_path( + @project.repository, + branch, + file, + literal_pathspec: true + ) + + content = blob_content(file, commit) + + # If the release has already been added (e.g. concurrently by another + # API call), we don't want to add it again. + break if content&.match?(release.header_start_pattern) + + service = Files::MultiService.new( + @project, + @user, + commit_message: message, + branch_name: branch, + start_branch: branch, + actions: [ + { + action: content ? 'update' : 'create', + content: Generator.new(content.to_s).add(release), + file_path: file, + last_commit_id: commit&.sha + } + ] + ) + + result = service.execute + + raise CommitError.new(result[:message]) if result[:status] != :success + end + end + + def blob_content(file, commit = nil) + return unless commit + + @project.repository.blob_at(commit.sha, file)&.data + end + end + end +end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb new file mode 100644 index 00000000000..3f06b612687 --- /dev/null +++ b/lib/gitlab/changelog/config.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Configuration settings used when generating changelogs. + class Config + ConfigError = Class.new(StandardError) + + # When rendering changelog entries, authors are not included. + AUTHORS_NONE = 'none' + + # The path to the configuration file as stored in the project's Git + # repository. + FILE_PATH = '.gitlab/changelog_config.yml' + + # The default date format to use for formatting release dates. + DEFAULT_DATE_FORMAT = '%Y-%m-%d' + + # The default template to use for generating release sections. + DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) + + attr_accessor :date_format, :categories, :template + + def self.from_git(project) + if (yaml = project.repository.changelog_config) + from_hash(project, YAML.safe_load(yaml)) + else + new(project) + end + end + + def self.from_hash(project, hash) + config = new(project) + + if (date = hash['date_format']) + config.date_format = date + end + + if (template = hash['template']) + # We use the full namespace here (and further down) as otherwise Rails + # may use the wrong constant when autoloading is used. + config.template = + ::Gitlab::Changelog::Template::Compiler.new.compile(template) + end + + if (categories = hash['categories']) + if categories.is_a?(Hash) + config.categories = categories + else + raise ConfigError, 'The "categories" configuration key must be a Hash' + end + end + + config + end + + def initialize(project) + @project = project + @date_format = DEFAULT_DATE_FORMAT + @template = + ::Gitlab::Changelog::Template::Compiler.new.compile(DEFAULT_TEMPLATE) + @categories = {} + end + + def contributor?(user) + @project.team.contributor?(user) + end + + def category(name) + @categories[name] || name + end + + def format_date(date) + date.strftime(@date_format) + end + end + end +end diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb new file mode 100644 index 00000000000..a80ca0728f9 --- /dev/null +++ b/lib/gitlab/changelog/generator.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Parsing and generating of Markdown changelogs. + class Generator + # The regex used to parse a release header. + RELEASE_REGEX = + /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze + + # The `input` argument must be a `String` containing the existing + # changelog Markdown. If no changelog exists, this should be an empty + # `String`. + def initialize(input = '') + @lines = input.lines + @locations = {} + + @lines.each_with_index do |line, index| + matches = line.match(RELEASE_REGEX) + + next if !matches || !matches[:version] + + @locations[matches[:version]] = index + end + end + + # Generates the Markdown for the given release and returns the new + # changelog Markdown content. + # + # The `release` argument must be an instance of + # `Gitlab::Changelog::Release`. + def add(release) + versions = [release.version, *@locations.keys] + + VersionSorter.rsort!(versions) + + new_index = versions.index(release.version) + new_lines = @lines.dup + markdown = release.to_markdown + + if (insert_after = versions[new_index + 1]) + line_index = @locations[insert_after] + + new_lines.insert(line_index, markdown) + else + # When adding to the end of the changelog, the previous section only + # has a single newline, resulting in the release section title + # following it immediately. When this is the case, we insert an extra + # empty line to keep the changelog readable in its raw form. + new_lines.push("\n") if versions.length > 1 + new_lines.push(markdown.rstrip) + new_lines.push("\n") + end + + new_lines.join + end + end + end +end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb new file mode 100644 index 00000000000..4c78eb5080c --- /dev/null +++ b/lib/gitlab/changelog/release.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A release to add to a changelog. + class Release + attr_reader :version + + def initialize(version:, date:, config:) + @version = version + @date = date + @config = config + @entries = Hash.new { |h, k| h[k] = [] } + + # This ensures that entries are presented in the same order as the + # categories Hash in the user's configuration. + @config.categories.values.each do |category| + @entries[category] = [] + end + end + + def add_entry( + title:, + commit:, + category:, + author: nil, + merge_request: nil + ) + # When changing these fields, keep in mind that this needs to be + # backwards compatible. For example, you can't just remove a field as + # this will break the changelog generation process for existing users. + entry = { + 'title' => title, + 'commit' => { + 'reference' => commit.to_reference(full: true), + 'trailers' => commit.trailers + } + } + + if author + entry['author'] = { + 'reference' => author.to_reference(full: true), + 'contributor' => @config.contributor?(author) + } + end + + if merge_request + entry['merge_request'] = { + 'reference' => merge_request.to_reference(full: true) + } + end + + @entries[@config.category(category)] << entry + end + + def to_markdown + # While not critical, we would like release sections to be separated by + # an empty line in the changelog; ensuring it's readable even in its + # raw form. + # + # Since it can be a bit tricky to get this right using Liquid, we + # enforce an empty line separator ourselves. + markdown = + @config.template.render('categories' => entries_for_template).strip + + # The release header can't be changed using the Liquid template, as we + # need this to be in a known format. Without this restriction, we won't + # know where to insert a new release section in an existing changelog. + "## #{@version} (#{release_date})\n\n#{markdown}\n\n" + end + + def header_start_pattern + /^##\s*#{Regexp.escape(@version)}/ + end + + private + + def release_date + @config.format_date(@date) + end + + def entries_for_template + @entries.map do |category, entries| + { + 'title' => category, + 'count' => entries.length, + 'single_change' => entries.length == 1, + 'entries' => entries + } + end + end + end + end +end diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl new file mode 100644 index 00000000000..838b7080f68 --- /dev/null +++ b/lib/gitlab/changelog/template.tpl @@ -0,0 +1,14 @@ +{% if categories %} +{% each categories %} +### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %}) + +{% each entries %} +- [{{ title }}]({{ commit.reference }})\ +{% if author.contributor %} by {{ author.reference }}{% end %}\ +{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %} +{% end %} + +{% end %} +{% else %} +No changes. +{% end %} diff --git a/lib/gitlab/changelog/template/compiler.rb b/lib/gitlab/changelog/template/compiler.rb new file mode 100644 index 00000000000..fa7724aa2da --- /dev/null +++ b/lib/gitlab/changelog/template/compiler.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + module Template + # Compiler is used for turning a minimal user templating language into an + # ERB template, without giving the user access to run arbitrary code. + # + # The template syntax is deliberately made as minimal as possible, and + # only supports the following: + # + # * Printing a value + # * Iterating over collections + # * if/else + # + # The syntax looks as follows: + # + # {% each users %} + # + # Name: {{user}} + # Likes cats: {% if likes_cats %}yes{% else %}no{% end %} + # + # {% end %} + # + # Newlines can be escaped by ending a line with a backslash. So this: + # + # foo \ + # bar + # + # Is the same as this: + # + # foo bar + # + # Templates are compiled into ERB templates, while taking care to make + # sure the user can't run arbitrary code. By using ERB we can let it do + # the heavy lifting of rendering data; all we need to provide is a + # translation layer. + # + # # Security + # + # The template syntax this compiler exposes is safe to be used by + # untrusted users. Not only are they unable to run arbitrary code, the + # compiler also enforces a limit on the integer sizes and the number of + # nested loops. ERB tags added by the user are also disabled. + class Compiler + # A pattern to match a single integer, with an upper size limit. + # + # We enforce a limit of 10 digits (= a 32 bits integer) so users can't + # trigger the allocation of infinitely large bignums, or trigger + # RangeError errors when using such integers to access an array value. + INTEGER = /^\d{1,10}$/.freeze + + # The name/path of a variable, such as `user.address.city`. + # + # It's important that this regular expression _doesn't_ allow for + # anything but letters, numbers, and underscores, otherwise a user may + # use those to "escape" our template and run arbirtary Ruby code. For + # example, take this variable: + # + # {{') ::Kernel.exit #'}} + # + # This would then be compiled into: + # + # <%= read(variables, '') ::Kernel.exit #'') %> + # + # Restricting the allowed characters makes this impossible. + VAR_NAME = /([\w\.]+)/.freeze + + # A variable tag, such as `{{username}}`. + VAR = /{{ \s* #{VAR_NAME} \s* }}/x.freeze + + # The opening tag for a statement. + STM_START = /{% \s*/x.freeze + + # The closing tag for a statement. + STM_END = /\s* %}/x.freeze + + # A regular `end` closing tag. + NORMAL_END = /#{STM_START} end #{STM_END}/x.freeze + + # An `end` closing tag on its own line, without any non-whitespace + # preceding or following it. + # + # These tags need some special care to make it easier to control + # whitespace. + LONELY_END = /^\s*#{NORMAL_END}\s$/x.freeze + + # An `else` tag. + ELSE = /#{STM_START} else #{STM_END}/x.freeze + + # The start of an `each` tag. + EACH = /#{STM_START} each \s+ #{VAR_NAME} #{STM_END}/x.freeze + + # The start of an `if` tag. + IF = /#{STM_START} if \s+ #{VAR_NAME} #{STM_END}/x.freeze + + # The pattern to use for escaping newlines. + ESCAPED_NEWLINE = /\\\n$/.freeze + + # The start tag for ERB tags. These tags will be escaped, preventing + # users from using ERB directly. + ERB_START_TAG = /<\\?\s*\\?\s*%/.freeze + + def compile(template) + transformed_lines = ['<% it = variables %>'] + + # ERB tags must be stripped here, otherwise a user may introduce ERB + # tags by making clever use of whitespace. See + # https://gitlab.com/gitlab-org/gitlab/-/issues/300224 for more + # information. + template = template.gsub(ERB_START_TAG, '<%%') + + template.each_line { |line| transformed_lines << transform(line) } + + # We use the full namespace here as otherwise Rails may use the wrong + # constant when autoloading is used. + ::Gitlab::Changelog::Template::Template.new(transformed_lines.join) + end + + def transform(line) + line.gsub!(ESCAPED_NEWLINE, '') + + # This replacement ensures that "end" blocks on their own lines + # don't add extra newlines. Using an ERB -%> tag sadly swallows too + # many newlines. + line.gsub!(LONELY_END, '<% end %>') + line.gsub!(NORMAL_END, '<% end %>') + line.gsub!(ELSE, '<% else -%>') + + line.gsub!(EACH) do + # No, `it; variables` isn't a syntax error. Using `;` marks + # `variables` as block-local, making it possible to re-assign it + # without affecting outer definitions of this variable. We use + # this to scope template variables to the right input Hash. + "<% each(#{read_path(Regexp.last_match(1))}) do |it; variables| -%><% variables = it -%>" + end + + line.gsub!(IF) { "<% if truthy?(#{read_path(Regexp.last_match(1))}) -%>" } + line.gsub!(VAR) { "<%= #{read_path(Regexp.last_match(1))} %>" } + line + end + + def read_path(path) + return path if path == 'it' + + args = path.split('.') + args.map! { |arg| arg.match?(INTEGER) ? "#{arg}" : "'#{arg}'" } + + "read(variables, #{args.join(', ')})" + end + end + end + end +end diff --git a/lib/gitlab/changelog/template/context.rb b/lib/gitlab/changelog/template/context.rb new file mode 100644 index 00000000000..8a0796d767e --- /dev/null +++ b/lib/gitlab/changelog/template/context.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + module Template + # Context is used to provide a binding/context to ERB templates used for + # rendering changelogs. + # + # This class extends BasicObject so that we only expose the bare minimum + # needed to render the ERB template. + class Context < BasicObject + MAX_NESTED_LOOPS = 4 + + def initialize(variables) + @variables = variables + @loop_nesting = 0 + end + + def get_binding + ::Kernel.binding + end + + def each(value, &block) + max = MAX_NESTED_LOOPS + + if @loop_nesting == max + ::Kernel.raise( + ::Template::TemplateError.new("You can only nest up to #{max} loops") + ) + end + + @loop_nesting += 1 + result = value.each(&block) if value.respond_to?(:each) + @loop_nesting -= 1 + + result + end + + # rubocop: disable Style/TrivialAccessors + def variables + @variables + end + # rubocop: enable Style/TrivialAccessors + + def read(source, *steps) + current = source + + steps.each do |step| + case current + when ::Hash + current = current[step] + when ::Array + return '' unless step.is_a?(::Integer) + + current = current[step] + else + break + end + end + + current + end + + def truthy?(value) + value.respond_to?(:any?) ? value.any? : !!value + end + end + end + end +end diff --git a/lib/gitlab/changelog/template/template.rb b/lib/gitlab/changelog/template/template.rb new file mode 100644 index 00000000000..0ff2852d6d4 --- /dev/null +++ b/lib/gitlab/changelog/template/template.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + module Template + # A wrapper around an ERB template user for rendering changelogs. + class Template + TemplateError = Class.new(StandardError) + + def initialize(erb) + # Don't change the trim mode, as this may require changes to the + # regular expressions used to turn the template syntax into ERB + # tags. + @erb = ERB.new(erb, trim_mode: '-') + end + + def render(data) + context = Context.new(data).get_binding + + # ERB produces a SyntaxError when processing templates, as it + # internally uses eval() for this. + @erb.result(context) + rescue SyntaxError + raise TemplateError.new("The template's syntax is invalid") + end + end + end + end +end diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 911f2993b8a..029a9210dc9 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -47,5 +47,13 @@ module Gitlab def self.kill Process.kill("KILL", Process.pid) end + + def self.run_gc + # Tenure any live objects from young-gen to old-gen + 4.times { GC.start(full_mark: false) } + # Run a full mark-and-sweep collection + GC.start + GC.stat + end end end diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/ci/badge/base.rb index fb55b9e2f1f..c65f120753d 100644 --- a/lib/gitlab/badge/base.rb +++ b/lib/gitlab/ci/badge/base.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge class Base def entity diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/ci/badge/coverage/metadata.rb index 9181ba2d4b0..7654b6d6fc5 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/ci/badge/coverage/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 390da014a5a..28863a0703b 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 1b985f83b22..7589fa5ff8b 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index b9ae68134b0..eec9fedfaa9 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract class for badge metadata diff --git a/lib/gitlab/badge/pipeline/metadata.rb b/lib/gitlab/ci/badge/pipeline/metadata.rb index d4d789558c9..2aa08476336 100644 --- a/lib/gitlab/badge/pipeline/metadata.rb +++ b/lib/gitlab/ci/badge/pipeline/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/ci/badge/pipeline/status.rb index f061ba22688..a2ee2642872 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/ci/badge/pipeline/status.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index af8e318395b..8430b01fc9a 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 9ac8f1c17f2..0580dad72ba 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract template class for badges diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 58adf6e506d..2aeb8453703 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -6,7 +6,7 @@ module Gitlab module Credentials class Base def type - self.class.name.demodulize.underscore + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index fa805abb8bb..e8996cb9dc4 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -20,7 +20,7 @@ module Gitlab end def providers - [Registry] + [Registry::GitlabRegistry, Registry::DependencyProxy] end end end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb deleted file mode 100644 index 1c8588d9913..00000000000 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Credentials - class Registry < Base - attr_reader :username, :password - - def initialize(build) - @username = 'gitlab-ci-token' - @password = build.token - end - - def url - Gitlab.config.registry.host_port - end - - def valid? - Gitlab.config.registry.enabled - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb new file mode 100644 index 00000000000..b6ac06cfb53 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class DependencyProxy < GitlabRegistry + def url + "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + end + + def valid? + Gitlab.config.dependency_proxy.enabled + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb new file mode 100644 index 00000000000..5bd30e677e9 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class GitlabRegistry < Credentials::Base + attr_reader :username, :password + + def initialize(build) + @username = Gitlab::Auth::CI_JOB_USER + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + + def type + 'registry' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a39afee194c..2d4f9cf635b 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -7,30 +7,17 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize Result = Struct.new(:when, :start_in, :allow_failure, :variables) do - def build_attributes(seed_attributes = {}) + def build_attributes { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure, - yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) + allow_failure: allow_failure }.compact end def pass? self.when != 'never' end - - private - - def yaml_variables(seed_variables) - return unless variables && seed_variables - - indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } - - variables.each_with_object(indexed_seed_variables) do |var, hash| - hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } - end.values - end end def initialize(rule_hashes, default_when:) diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 25fb9c0ca97..797193a6be5 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -31,9 +31,10 @@ module Gitlab current = @from while current <= @to - @labels << current.strftime(@format) - @total << (totals_count[current] || 0) - @success << (success_count[current] || 0) + label = current.strftime(@format) + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) current += interval_step end @@ -45,6 +46,7 @@ module Gitlab query .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) + .transform_keys { |date| date.strftime(@format) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 85e3514499c..a20b802be58 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script type image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - resource_group release secrets].freeze + release secrets].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -30,7 +30,6 @@ module Gitlab } validates :dependencies, array_of_strings: true - validates :resource_group, type: String validates :allow_failure, hash_or_boolean: true end @@ -124,7 +123,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :resource_group, + :interruptible, :timeout, :release, :allow_failure def self.matching?(name, config) @@ -174,7 +173,6 @@ module Gitlab ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - resource_group: resource_group, scheduling_type: needs_defined? ? :dag : :stage ).compact end @@ -186,8 +184,6 @@ module Gitlab private def allow_failure_criteria - return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? - if allow_failure_defined? && allow_failure_value.is_a?(Hash) allow_failure_value end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 5ef8cfbddb7..9584d19bdec 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs].freeze + inherit allow_failure when needs resource_group].freeze included do validations do @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :resource_group end def compose!(deps = nil) @@ -125,7 +126,8 @@ module Gitlab rules: rules_value, variables: root_and_job_variables_value, only: only_value, - except: except_value }.compact + except: except_value, + resource_group: resource_group }.compact end def root_and_job_variables_value diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 4d91cfd4c57..b85b7a9edeb 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -99,8 +99,6 @@ module Gitlab end def expand_variables(data) - return data unless ::Feature.enabled?(:variables_in_include_section_ci) - if data.is_a?(String) expand(data) else diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 7956cf14203..7155b60416b 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -55,21 +55,26 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) - end - def self.ci_pipeline_editor_page_enabled?(project) ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end - def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) - end - def self.rules_variables_enabled?(project) ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end + + def self.validate_build_dependencies?(project) + ::Feature.enabled?(:ci_validate_build_dependencies, default_enabled: :yaml) && + ::Feature.disabled?(:ci_validate_build_dependencies_override, project) + end + + def self.display_quality_on_mr_diff?(project) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) + end + + def self.display_codequality_backend_comparison?(project) + ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) + end end end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 985639982aa..2baa8faf849 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -20,6 +20,10 @@ module Gitlab rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end + + def self.instrument! + parsers.values.each { |parser_class| parser_class.prepend(Parsers::Instrumentation) } + end end end end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb new file mode 100644 index 00000000000..ab4a923d9aa --- /dev/null +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Instrumentation + BUCKETS = [0.25, 1, 5, 10].freeze + + def parse!(*args) + parser_result = nil + + duration = Benchmark.realtime do + parser_result = super + end + + labels = {} + + histogram = Gitlab::Metrics.histogram( + :ci_report_parser_duration_seconds, + 'Duration of parsing a CI report artifact', + labels, + BUCKETS + ) + + histogram.observe({ parser: self.class.name }, duration) + + parser_result + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 2ca51930c19..f0214bb4e38 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - pipelines + project.all_pipelines.ci_and_parent_sources .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,14 +33,6 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord - - def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) - project.all_pipelines.ci_and_parent_sources - else - project.ci_pipelines - end - end end end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index db6cca27f1c..c77f4dcca5a 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -45,6 +45,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def legacy_update_jobs_counter + strong_memoize(:legacy_update_jobs_counter) do + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fe3c2bca551..48411af6f38 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -159,7 +159,11 @@ module Gitlab next {} unless @using_rules if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_result.build_attributes(@seed_attributes) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) + + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) else rules_result.build_attributes end @@ -188,7 +192,6 @@ module Gitlab # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. def allow_failure_criteria_attributes - return {} unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? return {} if rules_attributes[:allow_failure].nil? return {} unless @seed_attributes.dig(:options, :allow_failure_criteria) diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb index c0641d9ff0a..794bd06be25 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb @@ -8,17 +8,17 @@ module Gitlab class ResourceGroup < Seed::Base include Gitlab::Utils::StrongMemoize - attr_reader :build, :resource_group_key + attr_reader :processable, :resource_group_key - def initialize(build, resource_group_key) - @build = build + def initialize(processable, resource_group_key) + @processable = processable @resource_group_key = resource_group_key end def to_resource return unless resource_group_key.present? - resource_group = build.project.resource_groups + resource_group = processable.project.resource_groups .safe_find_or_create_by(key: expanded_resource_group_key) resource_group if resource_group.persisted? @@ -28,7 +28,7 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) end end end diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb new file mode 100644 index 00000000000..e60a075e3f5 --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityMrDiff + attr_reader :files + + def initialize(raw_report) + @raw_report = raw_report + @files = {} + build_report! + end + + private + + def build_report! + codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + unless codequality_files[degradation.dig(:location, :path)].present? + codequality_files[degradation.dig(:location, :path)] = [] + end + + build_mr_diff_payload(codequality_files, degradation) + end + + @files = codequality_files + end + + def build_mr_diff_payload(codequality_files, degradation) + codequality_files[degradation.dig(:location, :path)] << { + line: degradation.dig(:location, :lines, :begin) || degradation.dig(:location, :positions, :begin, :line), + description: degradation[:description], + severity: degradation[:severity] + } + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 501d8737acd..daed75a42ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 192b1509fdc..6f30fc2dcd5 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" environment: name: production variables: diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb index 7cdb6a6c03c..92bed817875 100644 --- a/lib/gitlab/ci/trace/checksum.rb +++ b/lib/gitlab/ci/trace/checksum.rb @@ -30,7 +30,11 @@ module Gitlab end def state_crc32 - strong_memoize(:state_crc32) { build.pending_state&.crc32 } + strong_memoize(:state_crc32) do + ::Gitlab::Database::Consistency.with_read_consistency do + build.pending_state&.crc32 + end + end end def chunks_crc32 @@ -59,8 +63,10 @@ module Gitlab # def trace_chunks strong_memoize(:trace_chunks) do - build.trace_chunks.persisted - .select(::Ci::BuildTraceChunk.metadata_attributes) + ::Ci::BuildTraceChunk.with_read_consistency(build) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end end end diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 6f3e4ccf48d..7c2e39b1e53 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -227,12 +227,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def build_chunk - @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + def next_chunk + @chunks_cache[chunk_index] = begin + if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) + else + ::Ci::BuildTraceChunk + .new(build: build, chunk_index: chunk_index) + end + end end def ensure_chunk - current_chunk || build_chunk + current_chunk || next_chunk || current_chunk end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb new file mode 100644 index 00000000000..e2a54f90ecb --- /dev/null +++ b/lib/gitlab/ci/variables/helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Helpers + class << self + def merge_variables(current_vars, new_vars) + current_vars = transform_from_yaml_variables(current_vars) + new_vars = transform_from_yaml_variables(new_vars) + + transform_to_yaml_variables( + current_vars.merge(new_vars) + ) + end + + def transform_to_yaml_variables(vars) + vars.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def transform_from_yaml_variables(vars) + return vars.stringify_keys if vars.is_a?(Hash) + + vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 86749cda9c7..3459b69bebc 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,9 +123,7 @@ module Gitlab end def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 6d18f9070cc..48a1ab23fc2 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -12,10 +12,9 @@ module Gitlab VALID_NICENESS_LEVELS = %w{none realtime best-effort idle}.freeze attr_accessor :batch, :total_found, :total_cleaned - attr_reader :limit, :dry_run, :niceness, :logger + attr_reader :dry_run, :niceness, :logger - def initialize(limit: nil, dry_run: true, niceness: nil, logger: nil) - @limit = limit + def initialize(dry_run: true, niceness: nil, logger: nil) @dry_run = dry_run @niceness = (niceness || DEFAULT_NICENESS).downcase @logger = logger || Gitlab::AppLogger @@ -31,7 +30,11 @@ module Gitlab batch << artifact_file clean_batch! if batch.full? - break if limit_reached? + + if limit_reached? + log_info("Exiting due to reaching limit of #{limit}.") + break + end end clean_batch! @@ -128,6 +131,10 @@ module Gitlab def log_error(msg, params = {}) logger.error(msg) end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index a6638b2cbc8..99e7550629a 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -5,15 +5,14 @@ module Gitlab class OrphanLfsFileReferences include Gitlab::Utils::StrongMemoize - attr_reader :project, :dry_run, :logger, :limit + attr_reader :project, :dry_run, :logger DEFAULT_REMOVAL_LIMIT = 1000 - def initialize(project, dry_run: true, logger: nil, limit: nil) + def initialize(project, dry_run: true, logger: nil) @project = project @dry_run = dry_run @logger = logger || Gitlab::AppLogger - @limit = limit end def run! @@ -67,6 +66,10 @@ module Gitlab def log_info(msg) logger.info("#{'[DRY RUN] ' if dry_run}#{msg}") end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 4ae75e0db0a..3c71ca9fcf0 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../utils' # Gitlab::Utils + module Gitlab module Cluster # @@ -64,6 +66,10 @@ module Gitlab # Blocks will be executed in the order in which they are registered. # class LifecycleEvents + FatalError = Class.new(Exception) # rubocop:disable Lint/InheritException + + USE_FATAL_LIFECYCLE_EVENTS = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_FATAL_LIFECYCLE_EVENTS', 'true')) + class << self # # Hook registration methods (called from initializers) @@ -111,24 +117,24 @@ module Gitlab # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # def do_worker_start - call(@worker_start_hooks) + call(:worker_start_hooks, @worker_start_hooks) end def do_before_fork - call(@before_fork_hooks) + call(:before_fork_hooks, @before_fork_hooks) end def do_before_graceful_shutdown - call(@master_blackout_period) + call(:master_blackout_period, @master_blackout_period) blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i sleep(blackout_seconds) if blackout_seconds > 0 - call(@master_graceful_shutdown) + call(:master_graceful_shutdown, @master_graceful_shutdown) end def do_before_master_restart - call(@master_restart_hooks) + call(:master_restart_hooks, @master_restart_hooks) end # DEPRECATED @@ -143,8 +149,18 @@ module Gitlab private - def call(hooks) - hooks&.each(&:call) + def call(name, hooks) + return unless hooks + + hooks.each do |hook| + hook.call + rescue => e + Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook) + warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".") + + # we consider lifecycle hooks to be fatal errors + raise FatalError, e if USE_FATAL_LIFECYCLE_EVENTS + end end def in_clustered_environment? diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 822012e0ed6..fd9f58a34f3 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -35,6 +35,10 @@ module Gitlab # regularly rather than rely on OOM behavior for periodic restarting. config.rolling_restart_frequency = 43200 # 12 hours in seconds. + # Spread the rolling restarts out over 1 hour to avoid too many simultaneous + # process startups. + config.rolling_restart_splay_seconds = 0.0..3600.0 # 0 to 1 hour in seconds. + observer = Gitlab::Cluster::PumaWorkerKillerObserver.new config.pre_term = observer.callback end diff --git a/lib/gitlab/composer/cache.rb b/lib/gitlab/composer/cache.rb new file mode 100644 index 00000000000..1f404d63047 --- /dev/null +++ b/lib/gitlab/composer/cache.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'tempfile' + +module Gitlab + module Composer + class Cache + def initialize(project:, name:, last_page_sha: nil) + @project = project + @name = name + @last_page_sha = last_page_sha + end + + def execute + Packages::Composer::Metadatum.transaction do # rubocop: disable CodeReuse/ActiveRecord + # make sure we lock these records at the start + locked_package_metadata + + if locked_package_metadata.any? + mark_pages_for_delete(shas_to_delete) + + create_cache_page! + + # assign the newest page SHA to the packages + locked_package_metadata.update_all(version_cache_sha: version_index.sha) + elsif @last_page_sha + mark_pages_for_delete([@last_page_sha]) + end + end + end + + private + + def mark_pages_for_delete(shas) + Packages::Composer::CacheFile + .with_namespace(@project.namespace) + .with_sha(shas) + .update_all(delete_at: 1.day.from_now) + end + + def create_cache_page! + Packages::Composer::CacheFile + .safe_find_or_create_by!(namespace_id: @project.namespace_id, file_sha256: version_index.sha) do |cache_file| + cache_file.file = CarrierWaveStringFile.new(version_index.to_json) + end + end + + def version_index + @version_index ||= ::Gitlab::Composer::VersionIndex.new(siblings) + end + + def siblings + @siblings ||= locked_package_metadata.map(&:package) + end + + # find all metadata of the package versions and lock it for update + def locked_package_metadata + @locked_package_metadata ||= Packages::Composer::Metadatum + .for_package(@name, @project.id) + .locked_for_update + end + + def shas_to_delete + locked_package_metadata + .map(&:version_cache_sha) + .reject { |sha| sha == version_index.sha } + .compact + end + end + end +end diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index de9a17a453f..ac0071cdc53 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -20,7 +20,7 @@ module Gitlab private def package_versions_map - @packages.each_with_object({}) do |package, map| + @packages.sort_by(&:version).each_with_object({}) do |package, map| map[package.version] = package_metadata(package) end end diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index 7526c10b608..d03997b4158 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -35,7 +35,7 @@ module Gitlab def secret OpenSSL::HMAC.hexdigest( - OpenSSL::Digest::SHA256.new, + OpenSSL::Digest.new('SHA256'), ::Settings.attr_encrypted_db_key_base, HMAC_KEY ) diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 87a03d9c58f..4428354642d 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,25 +6,44 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32, - iv: Settings.attr_encrypted_db_key_base_12 + key: Settings.attr_encrypted_db_key_base_32 }.freeze + AES256_GCM_IV_STATIC = Settings.attr_encrypted_db_key_base_12 + def sha256(value) salt = Settings.attr_encrypted_db_key_base_truncated ::Digest::SHA256.base64digest("#{value}#{salt}") end - def aes256_gcm_encrypt(value) - encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.strict_encode64(encrypted_token) + def aes256_gcm_encrypt(value, nonce: nil) + aes256_gcm_encrypt_using_static_nonce(value) end def aes256_gcm_decrypt(value) return unless value + nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC encrypted_token = Base64.decode64(value) - Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token)) + decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) + decrypted_token + end + + def dynamic_nonce(value) + TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC + end + + def aes256_gcm_encrypt_using_static_nonce(value) + create_encrypted_token(value, AES256_GCM_IV_STATIC) + end + + def read_only? + Gitlab::Database.read_only? + end + + def create_encrypted_token(value, iv) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv)) + Base64.strict_encode64(encrypted_token) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index d0579a44219..0bf41f9dc0d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def current_application_settings? + Gitlab::SafeRequestStore.exist?(:current_application_settings) || ::ApplicationSetting.current.present? + end + def expire_current_application_settings ::ApplicationSetting.expire Gitlab::SafeRequestStore.delete(:current_application_settings) diff --git a/lib/gitlab/danger/base_linter.rb b/lib/gitlab/danger/base_linter.rb deleted file mode 100644 index 898434724bd..00000000000 --- a/lib/gitlab/danger/base_linter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - class BaseLinter - MIN_SUBJECT_WORDS_COUNT = 3 - MAX_LINE_LENGTH = 72 - - attr_reader :commit, :problems - - def self.problems_mapping - { - subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", - subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", - subject_starts_with_lowercase: "The %s must start with a capital letter", - subject_ends_with_a_period: "The %s must not end with a period" - } - end - - def self.subject_description - 'commit subject' - end - - def initialize(commit) - @commit = commit - @problems = {} - end - - def failed? - problems.any? - end - - def add_problem(problem_key, *args) - @problems[problem_key] = sprintf(self.class.problems_mapping[problem_key], *args) - end - - def lint_subject - if subject_too_short? - add_problem(:subject_too_short, self.class.subject_description) - end - - if subject_too_long? - add_problem(:subject_too_long, self.class.subject_description) - end - - if subject_starts_with_lowercase? - add_problem(:subject_starts_with_lowercase, self.class.subject_description) - end - - if subject_ends_with_a_period? - add_problem(:subject_ends_with_a_period, self.class.subject_description) - end - - self - end - - private - - def subject - TitleLinting.remove_draft_flag(message_parts[0]) - end - - def subject_too_short? - subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT - end - - def subject_too_long? - line_too_long?(subject) - end - - def line_too_long?(line) - line.length > MAX_LINE_LENGTH - end - - def subject_starts_with_lowercase? - return false if ('A'..'Z').cover?(subject[0]) - - first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0] - first_char_downcased = first_char.downcase - return true unless ('a'..'z').cover?(first_char_downcased) - - first_char.downcase == first_char - end - - def subject_ends_with_a_period? - subject.end_with?('.') - end - - def message_parts - @message_parts ||= commit.message.split("\n", 3) - end - end - end -end diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb deleted file mode 100644 index 4b85775ed98..00000000000 --- a/lib/gitlab/danger/changelog.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - module Changelog - NO_CHANGELOG_LABELS = [ - 'tooling', - 'tooling::pipelines', - 'tooling::workflow', - 'ci-build', - 'meta' - ].freeze - NO_CHANGELOG_CATEGORIES = %i[docs none].freeze - CREATE_CHANGELOG_COMMAND = 'bin/changelog -m %<mr_iid>s "%<mr_title>s"' - CREATE_EE_CHANGELOG_COMMAND = 'bin/changelog --ee -m %<mr_iid>s "%<mr_title>s"' - CHANGELOG_MODIFIED_URL_TEXT = "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n" - CHANGELOG_MISSING_URL_TEXT = "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n" - - OPTIONAL_CHANGELOG_MESSAGE = <<~MSG - If you want to create a changelog entry for GitLab FOSS, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - If you want to create a changelog entry for GitLab EE, run the following instead: - - #{CREATE_EE_CHANGELOG_COMMAND} - - If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message. - MSG - - REQUIRED_CHANGELOG_MESSAGE = <<~MSG - To create a changelog entry, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - This merge request requires a changelog entry because it [introduces a database migration](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry). - MSG - - def required? - git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } - end - alias_method :db_changes?, :required? - - def optional? - categories_need_changelog? && without_no_changelog_label? - end - - def found - @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } - end - - def ee_changelog? - found.start_with?('ee/') - end - - def modified_text - CHANGELOG_MODIFIED_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def required_text - CHANGELOG_MISSING_URL_TEXT + - format(REQUIRED_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def optional_text - CHANGELOG_MISSING_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - private - - def mr_iid - gitlab.mr_json["iid"] - end - - def sanitized_mr_title - TitleLinting.sanitize_mr_title(gitlab.mr_json["title"]) - end - - def categories_need_changelog? - (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any? - end - - def without_no_changelog_label? - (gitlab.mr_labels & NO_CHANGELOG_LABELS).empty? - end - end - end -end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb deleted file mode 100644 index e23f5900433..00000000000 --- a/lib/gitlab/danger/commit_linter.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: true - -emoji_checker_path = File.expand_path('emoji_checker', __dir__) -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) - require_dependency(emoji_checker_path) -else - require_relative(base_linter_path) - require_relative(emoji_checker_path) -end - -module Gitlab - module Danger - class CommitLinter < BaseLinter - MAX_CHANGED_FILES_IN_COMMIT = 3 - MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze - - def self.problems_mapping - super.merge( - { - separator_missing: "The commit subject and body must be separated by a blank line", - details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \ - "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body", - details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line", - message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \ - "to the commit message, and are displayed as plain text outside of GitLab", - message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \ - "message, and may not be displayed properly everywhere", - message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \ - "`!123`), as short references are displayed as plain text outside of GitLab" - } - ) - end - - def initialize(commit) - super - - @linted = false - end - - def fixup? - commit.message.start_with?('fixup!', 'squash!') - end - - def suggestion? - commit.message.start_with?('Apply suggestion to') - end - - def merge? - commit.message.start_with?('Merge branch') - end - - def revert? - commit.message.start_with?('Revert "') - end - - def multi_line? - !details.nil? && !details.empty? - end - - def lint - return self if @linted - - @linted = true - lint_subject - lint_separator - lint_details - lint_message - - self - end - - private - - def lint_separator - return self unless separator && !separator.empty? - - add_problem(:separator_missing) - - self - end - - def lint_details - if !multi_line? && many_changes? - add_problem(:details_too_many_changes) - end - - details&.each_line do |line| - line_without_urls = line.strip.gsub(%r{https?://\S+}, '') - - # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but - # only if the line _without_ the URL does not exceed this limit. - next unless line_too_long?(line_without_urls) - - add_problem(:details_line_too_long) - break - end - - self - end - - def lint_message - if message_contains_text_emoji? - add_problem(:message_contains_text_emoji) - end - - if message_contains_unicode_emoji? - add_problem(:message_contains_unicode_emoji) - end - - if message_contains_short_reference? - add_problem(:message_contains_short_reference) - end - - self - end - - def files_changed - commit.diff_parent.stats[:total][:files] - end - - def lines_changed - commit.diff_parent.stats[:total][:lines] - end - - def many_changes? - files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT - end - - def separator - message_parts[1] - end - - def details - message_parts[2]&.gsub(/^Signed-off-by.*$/, '') - end - - def message_contains_text_emoji? - emoji_checker.includes_text_emoji?(commit.message) - end - - def message_contains_unicode_emoji? - emoji_checker.includes_unicode_emoji?(commit.message) - end - - def message_contains_short_reference? - commit.message.match?(SHORT_REFERENCE_REGEX) - end - - def emoji_checker - @emoji_checker ||= Gitlab::Danger::EmojiChecker.new - end - end - end -end diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb deleted file mode 100644 index e31a6ae5011..00000000000 --- a/lib/gitlab/danger/emoji_checker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module Gitlab - module Danger - class EmojiChecker - DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__) - ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__) - - # A regex that indicates a piece of text _might_ include an Emoji. The regex - # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this - # regex to save us from having to check for all possible emoji names when we - # know one definitely is not included. - LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze - - UNICODE_EMOJI_REGEX = %r{( - [\u{1F300}-\u{1F5FF}] | - [\u{1F1E6}-\u{1F1FF}] | - [\u{2700}-\u{27BF}] | - [\u{1F900}-\u{1F9FF}] | - [\u{1F600}-\u{1F64F}] | - [\u{1F680}-\u{1F6FF}] | - [\u{2600}-\u{26FF}] - )}x.freeze - - def initialize - names = JSON.parse(File.read(DIGESTS)).keys + - JSON.parse(File.read(ALIASES)).keys - - @emoji = names.map { |name| ":#{name}:" } - end - - def includes_text_emoji?(text) - return false unless text.match?(LIKELY_EMOJI) - - @emoji.any? { |emoji| text.include?(emoji) } - end - - def includes_unicode_emoji?(text) - text.match?(UNICODE_EMOJI_REGEX) - end - end - end -end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb deleted file mode 100644 index 09e013e24b8..00000000000 --- a/lib/gitlab/danger/helper.rb +++ /dev/null @@ -1,273 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'title_linting' - -module Gitlab - module Danger - module Helper - RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - - # Returns a list of all files that have been added, modified or renamed. - # `git.modified_files` might contain paths that already have been renamed, - # so we need to remove them from the list. - # - # Considering these changes: - # - # - A new_file.rb - # - D deleted_file.rb - # - M modified_file.rb - # - R renamed_file_before.rb -> renamed_file_after.rb - # - # it will return - # ``` - # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] - # ``` - # - # @return [Array<String>] - def all_changed_files - Set.new - .merge(git.added_files.to_a) - .merge(git.modified_files.to_a) - .merge(git.renamed_files.map { |x| x[:after] }) - .subtract(git.renamed_files.map { |x| x[:before] }) - .to_a - .sort - end - - # Returns a string containing changed lines as git diff - # - # Considering changing a line in lib/gitlab/usage_data.rb it will return: - # - # [ "--- a/lib/gitlab/usage_data.rb", - # "+++ b/lib/gitlab/usage_data.rb", - # "+ # Test change", - # "- # Old change" ] - def changed_lines(changed_file) - diff = git.diff_for_file(changed_file) - return [] unless diff - - diff.patch.split("\n").select { |line| %r{^[+-]}.match?(line) } - end - - def all_ee_changes - all_changed_files.grep(%r{\Aee/}) - end - - def ee? - # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) - end - - def gitlab_helper - # Unfortunately the following does not work: - # - respond_to?(:gitlab) - # - respond_to?(:gitlab, true) - gitlab - rescue NameError - nil - end - - def release_automation? - gitlab_helper&.mr_author == RELEASE_TOOLS_BOT - end - - def project_name - ee? ? 'gitlab' : 'gitlab-foss' - end - - def markdown_list(items) - list = items.map { |item| "* `#{item}`" }.join("\n") - - if items.size > 10 - "\n<details>\n\n#{list}\n\n</details>\n" - else - list - end - end - - # @return [Hash<String,Array<String>>] - def changes_by_category - all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| - categories_for_file(file).each { |category| hash[category] << file } - end - end - - # Determines the categories a file is in, e.g., `[:frontend]`, `[:backend]`, or `%i[frontend engineering_productivity]` - # using filename regex and specific change regex if given. - # - # @return Array<Symbol> - def categories_for_file(file) - _, categories = CATEGORIES.find do |key, _| - filename_regex, changes_regex = Array(key) - - found = filename_regex.match?(file) - found &&= changed_lines(file).any? { |changed_line| changes_regex.match?(changed_line) } if changes_regex - - found - end - - Array(categories || :unknown) - end - - # Returns the GFM for a category label, making its best guess if it's not - # a category we know about. - # - # @return[String] - def label_for_category(category) - CATEGORY_LABELS.fetch(category, "~#{category}") - end - - CATEGORY_LABELS = { - docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. - none: "", - qa: "~QA", - test: "~test ~Quality for `spec/features/*`", - engineering_productivity: '~"Engineering Productivity" for CI, Danger', - ci_template: '~"ci::templates"' - }.freeze - # First-match win, so be sure to put more specific regex at the top... - CATEGORIES = { - [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend], - - %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, - %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, - - %r{\A(ee/)?app/(assets|views)/} => :frontend, - %r{\A(ee/)?public/} => :frontend, - %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, - %r{\A(ee/)?vendor/assets/} => :frontend, - %r{\A(ee/)?scripts/frontend/} => :frontend, - %r{(\A|/)( - \.babelrc | - \.eslintignore | - \.eslintrc(\.yml)? | - \.nvmrc | - \.prettierignore | - \.prettierrc | - \.scss-lint.yml | - \.stylelintrc | - \.haml-lint.yml | - \.haml-lint_todo.yml | - babel\.config\.js | - jest\.config\.js | - package\.json | - yarn\.lock | - config/.+\.js - )\z}x => :frontend, - - %r{(\A|/)( - \.gitlab/ci/frontend\.gitlab-ci\.yml - )\z}x => %i[frontend engineering_productivity], - - %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, - %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, - %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, - %r{\A(ee/)?app/finders/} => :database, - %r{\Arubocop/cop/migration(/|\.rb)} => :database, - - %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, - %r{\A\.codeclimate\.yml\z} => :engineering_productivity, - %r{\Alefthook.yml\z} => :engineering_productivity, - %r{\A\.editorconfig\z} => :engineering_productivity, - %r{Dangerfile\z} => :engineering_productivity, - %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, - %r{\A(ee/)?scripts/} => :engineering_productivity, - %r{\Atooling/} => :engineering_productivity, - %r{(CODEOWNERS)} => :engineering_productivity, - %r{(tests.yml)} => :engineering_productivity, - - %r{\Alib/gitlab/ci/templates} => :ci_template, - - %r{\A(ee/)?spec/features/} => :test, - %r{\A(ee/)?spec/support/shared_examples/features/} => :test, - %r{\A(ee/)?spec/support/shared_contexts/features/} => :test, - %r{\A(ee/)?spec/support/helpers/features/} => :test, - - %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, - %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, - %r{\A(ee/)?spec/} => :backend, - %r{\A(ee/)?vendor/} => :backend, - %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, - %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend, - %r{\Afile_hooks/} => :backend, - - %r{\A(ee/)?qa/} => :qa, - - # Files that don't fit into any category are marked with :none - %r{\A(ee/)?changelogs/} => :none, - %r{\Alocale/gitlab\.pot\z} => :none, - %r{\Adata/whats_new/} => :none, - - # GraphQL auto generated doc files and schema - %r{\Adoc/api/graphql/reference/} => :backend, - - # Fallbacks in case the above patterns miss anything - %r{\.rb\z} => :backend, - %r{( - \.(md|txt)\z | - \.markdownlint\.json - )}x => :none, # To reinstate roulette for documentation, set to `:docs`. - %r{\.js\z} => :frontend - }.freeze - - def new_teammates(usernames) - usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } - end - - def draft_mr? - return false unless gitlab_helper - - TitleLinting.has_draft_flag?(gitlab_helper.mr_json['title']) - end - - def security_mr? - return false unless gitlab_helper - - gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/') - end - - def cherry_pick_mr? - return false unless gitlab_helper - - /cherry[\s-]*pick/i.match?(gitlab_helper.mr_json['title']) - end - - def stable_branch? - return false unless gitlab_helper - - /\A\d+-\d+-stable-ee/i.match?(gitlab_helper.mr_json['target_branch']) - end - - def mr_has_labels?(*labels) - return false unless gitlab_helper - - labels = labels.flatten.uniq - (labels & gitlab_helper.mr_labels) == labels - end - - def labels_list(labels, sep: ', ') - labels.map { |label| %Q{~"#{label}"} }.join(sep) - end - - def prepare_labels_for_mr(labels) - return '' unless labels.any? - - "/label #{labels_list(labels, sep: ' ')}" - end - - def changed_files(regex) - all_changed_files.grep(regex) - end - - def has_database_scoped_labels?(current_mr_labels) - current_mr_labels.any? { |label| label.start_with?('database::') } - end - - def has_ci_changes? - changed_files(%r{\A(\.gitlab-ci\.yml|\.gitlab/ci/)}).any? - end - end - end -end diff --git a/lib/gitlab/danger/merge_request_linter.rb b/lib/gitlab/danger/merge_request_linter.rb deleted file mode 100644 index ed354bfc68d..00000000000 --- a/lib/gitlab/danger/merge_request_linter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) -else - require_relative(base_linter_path) -end - -module Gitlab - module Danger - class MergeRequestLinter < BaseLinter - alias_method :lint, :lint_subject - - def self.subject_description - 'merge request title' - end - - def self.mr_run_options_regex - [ - 'RUN AS-IF-FOSS', - 'UPDATE CACHE', - 'RUN ALL RSPEC', - 'SKIP RSPEC FAIL-FAST' - ].join('|') - end - - private - - def subject - super.gsub(/\[?(#{self.class.mr_run_options_regex})\]?/, '').strip - end - end - end -end diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb deleted file mode 100644 index 06da4ed9ad3..00000000000 --- a/lib/gitlab/danger/request_helper.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'json' - -module Gitlab - module Danger - module RequestHelper - HTTPError = Class.new(RuntimeError) - - # @param [String] url - def self.http_get_json(url) - rsp = Net::HTTP.get_response(URI.parse(url)) - - unless rsp.is_a?(Net::HTTPOK) - raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" - end - - JSON.parse(rsp.body) - end - end - end -end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb deleted file mode 100644 index 21feda2cf20..00000000000 --- a/lib/gitlab/danger/roulette.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) -require_relative 'weightage/reviewers' -require_relative 'weightage/maintainers' - -module Gitlab - module Danger - module Roulette - ROULETTE_DATA_URL = 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json' - HOURS_WHEN_PERSON_CAN_BE_PICKED = (6..14).freeze - - INCLUDE_TIMEZONE_FOR_CATEGORY = { - database: false - }.freeze - - Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment) - - def team_mr_author - team.find { |person| person.username == mr_author_username } - end - - # Assigns GitLab team members to be reviewer and maintainer - # for each change category that a Merge Request contains. - # - # @return [Array<Spin>] - def spin(project, categories, timezone_experiment: false) - spins = categories.sort.map do |category| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment) - - spin_for_category(project, category, timezone_experiment: including_timezone) - end - - backend_spin = spins.find { |spin| spin.category == :backend } - - spins.each do |spin| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment) - case spin.category - when :qa - # MR includes QA changes, but also other changes, and author isn't an SET - if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, []) - spin.optional_role = :maintainer - end - when :test - spin.optional_role = :maintainer - - if spin.reviewer.nil? - # Fetch an already picked backend reviewer, or pick one otherwise - spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer - end - when :engineering_productivity - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - when :ci_template - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - end - end - - spins - end - - # Looks up the current list of GitLab team members and parses it into a - # useful form - # - # @return [Array<Teammate>] - def team - @team ||= - begin - data = Gitlab::Danger::RequestHelper.http_get_json(ROULETTE_DATA_URL) - data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } - rescue JSON::ParserError - raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" - end - end - - # Like +team+, but only returns teammates in the current project, based on - # project_name. - # - # @return [Array<Teammate>] - def project_team(project_name) - team.select { |member| member.in_project?(project_name) } - rescue => err - warn("Reviewer roulette failed to load team data: #{err.message}") - [] - end - - # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the - # selection will change on next spin - # @param [Array<Teammate>] people - def spin_for_person(people, random:, timezone_experiment: false) - shuffled_people = people.shuffle(random: random) - - if timezone_experiment - shuffled_people.find(&method(:valid_person_with_timezone?)) - else - shuffled_people.find(&method(:valid_person?)) - end - end - - private - - # @param [Teammate] person - # @return [Boolean] - def valid_person?(person) - !mr_author?(person) && person.available - end - - # @param [Teammate] person - # @return [Boolean] - def valid_person_with_timezone?(person) - valid_person?(person) && HOURS_WHEN_PERSON_CAN_BE_PICKED.cover?(person.local_hour) - end - - # @param [Teammate] person - # @return [Boolean] - def mr_author?(person) - person.username == mr_author_username - end - - def mr_author_username - helper.gitlab_helper&.mr_author || `whoami` - end - - def mr_source_branch - return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json - - helper.gitlab_helper.mr_json['source_branch'] - end - - def mr_labels - helper.gitlab_helper&.mr_labels || [] - end - - def new_random(seed) - Random.new(Digest::MD5.hexdigest(seed).to_i(16)) - end - - def spin_role_for_category(team, role, project, category) - team.select do |member| - member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def spin_for_category(project, category, timezone_experiment: false) - team = project_team(project) - reviewers, traintainers, maintainers = - %i[reviewer traintainer maintainer].map do |role| - spin_role_for_category(team, role, project, category) - end - - random = new_random(mr_source_branch) - - weighted_reviewers = Weightage::Reviewers.new(reviewers, traintainers).execute - weighted_maintainers = Weightage::Maintainers.new(maintainers).execute - - reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) - maintainer = spin_for_person(weighted_maintainers, random: random, timezone_experiment: timezone_experiment) - - Spin.new(category, reviewer, maintainer, false, timezone_experiment) - end - end - end -end diff --git a/lib/gitlab/danger/sidekiq_queues.rb b/lib/gitlab/danger/sidekiq_queues.rb deleted file mode 100644 index 726b6134abf..00000000000 --- a/lib/gitlab/danger/sidekiq_queues.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module SidekiqQueues - def changed_queue_files - @changed_queue_files ||= git.modified_files.grep(%r{\A(ee/)?app/workers/all_queues\.yml}) - end - - def added_queue_names - @added_queue_names ||= new_queues.keys - old_queues.keys - end - - def changed_queue_names - @changed_queue_names ||= - (new_queues.values_at(*old_queues.keys) - old_queues.values) - .compact.map { |queue| queue[:name] } - end - - private - - def old_queues - @old_queues ||= queues_for(gitlab.base_commit) - end - - def new_queues - @new_queues ||= queues_for(gitlab.head_commit) - end - - def queues_for(branch) - changed_queue_files - .flat_map { |file| YAML.safe_load(`git show #{branch}:#{file}`, permitted_classes: [Symbol]) } - .to_h { |queue| [queue[:name], queue] } - end - end - end -end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb deleted file mode 100644 index 911b84d93ec..00000000000 --- a/lib/gitlab/danger/teammate.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :reduced_capacity, :tz_offset_hours - - # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb - def initialize(options = {}) - @options = options - @username = options['username'] - @name = options['name'] - @markdown_name = options['markdown_name'] - @role = options['role'] - @projects = options['projects'] - @available = options['available'] - @hungry = options['hungry'] - @reduced_capacity = options['reduced_capacity'] - @tz_offset_hours = options['tz_offset_hours'] - end - - def to_h - options - end - - def ==(other) - return false unless other.respond_to?(:username) - - other.username == username - end - - def in_project?(name) - projects&.has_key?(name) - end - - def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) - end - - def traintainer?(project, category, labels) - has_capability?(project, category, :trainee_maintainer, labels) - end - - def maintainer?(project, category, labels) - has_capability?(project, category, :maintainer, labels) - end - - def markdown_name(author: nil) - "#{@markdown_name} (#{utc_offset_text(author)})" - end - - def local_hour - (Time.now.utc + tz_offset_hours * 3600).hour - end - - protected - - def floored_offset_hours - floored_offset = tz_offset_hours.floor(0) - - floored_offset == tz_offset_hours ? floored_offset : tz_offset_hours - end - - private - - def utc_offset_text(author = nil) - offset_text = - if floored_offset_hours >= 0 - "UTC+#{floored_offset_hours}" - else - "UTC#{floored_offset_hours}" - end - - return offset_text unless author - - "#{offset_text}, #{offset_diff_compared_to_author(author)}" - end - - def offset_diff_compared_to_author(author) - diff = floored_offset_hours - author.floored_offset_hours - return "same timezone as `@#{author.username}`" if diff == 0 - - ahead_or_behind = diff < 0 ? 'behind' : 'ahead of' - pluralized_hours = pluralize(diff.abs, 'hour', 'hours') - - "#{pluralized_hours} #{ahead_or_behind} `@#{author.username}`" - end - - def has_capability?(project, category, kind, labels) - case category - when :test - area = role[/Software Engineer in Test(?:.*?, (\w+))/, 1] - - area && labels.any?("devops::#{area.downcase}") if kind == :reviewer - when :engineering_productivity - return false unless role[/Engineering Productivity/] - return true if kind == :reviewer - return true if capabilities(project).include?("#{kind} engineering_productivity") - - capabilities(project).include?("#{kind} backend") - else - capabilities(project).include?("#{kind} #{category}") - end - end - - def capabilities(project) - Array(projects.fetch(project, [])) - end - - def pluralize(count, singular, plural) - word = count == 1 || count.to_s =~ /^1(\.0+)?$/ ? singular : plural - - "#{count || 0} #{word}" - end - end - end -end diff --git a/lib/gitlab/danger/title_linting.rb b/lib/gitlab/danger/title_linting.rb deleted file mode 100644 index db1ccaaf9a9..00000000000 --- a/lib/gitlab/danger/title_linting.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module TitleLinting - DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze - - module_function - - def sanitize_mr_title(title) - remove_draft_flag(title).gsub(/`/, '\\\`') - end - - def remove_draft_flag(title) - title.gsub(DRAFT_REGEX, '') - end - - def has_draft_flag?(title) - DRAFT_REGEX.match?(title) - end - end - end -end diff --git a/lib/gitlab/danger/weightage.rb b/lib/gitlab/danger/weightage.rb deleted file mode 100644 index 67fade27573..00000000000 --- a/lib/gitlab/danger/weightage.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module Weightage - CAPACITY_MULTIPLIER = 2 # change this number to change what it means to be a reduced capacity reviewer 1/this number - BASE_REVIEWER_WEIGHT = 1 - end - end -end diff --git a/lib/gitlab/danger/weightage/maintainers.rb b/lib/gitlab/danger/weightage/maintainers.rb deleted file mode 100644 index cc0eb370e7a..00000000000 --- a/lib/gitlab/danger/weightage/maintainers.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - class Maintainers - def initialize(maintainers) - @maintainers = maintainers - end - - def execute - maintainers.each_with_object([]) do |maintainer, weighted_maintainers| - add_weighted_reviewer(weighted_maintainers, maintainer, BASE_REVIEWER_WEIGHT) - end - end - - private - - attr_reader :maintainers - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/danger/weightage/reviewers.rb b/lib/gitlab/danger/weightage/reviewers.rb deleted file mode 100644 index c8019be716e..00000000000 --- a/lib/gitlab/danger/weightage/reviewers.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - # Weights after (current multiplier of 2) - # - # +------------------------------+--------------------------------+ - # | reviewer type | weight(times in reviewer pool) | - # +------------------------------+--------------------------------+ - # | reduced capacity reviewer | 1 | - # | reviewer | 2 | - # | hungry reviewer | 4 | - # | reduced capacity traintainer | 3 | - # | traintainer | 6 | - # | hungry traintainer | 8 | - # +------------------------------+--------------------------------+ - # - class Reviewers - DEFAULT_REVIEWER_WEIGHT = CAPACITY_MULTIPLIER * BASE_REVIEWER_WEIGHT - TRAINTAINER_WEIGHT = 3 - - def initialize(reviewers, traintainers) - @reviewers = reviewers - @traintainers = traintainers - end - - def execute - # TODO: take CODEOWNERS into account? - # https://gitlab.com/gitlab-org/gitlab/issues/26723 - - weighted_reviewers + weighted_traintainers - end - - private - - attr_reader :reviewers, :traintainers - - def weighted_reviewers - reviewers.each_with_object([]) do |reviewer, total_reviewers| - add_weighted_reviewer(total_reviewers, reviewer, BASE_REVIEWER_WEIGHT) - end - end - - def weighted_traintainers - traintainers.each_with_object([]) do |reviewer, total_traintainers| - add_weighted_reviewer(total_traintainers, reviewer, TRAINTAINER_WEIGHT) - end - end - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - elsif reviewer.hungry - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER + DEFAULT_REVIEWER_WEIGHT) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e6702c5a38b..e17bd25e57e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -82,7 +82,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 14facd6b1d4..2413f68f4d0 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -76,7 +76,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb new file mode 100644 index 00000000000..b7d06a26ddb --- /dev/null +++ b/lib/gitlab/database/consistency.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + ## + # This class is used to make it possible to ensure read consistency in + # GitLab EE without the need of overriding a lot of methods / classes / + # classs. + # + # This is a CE class that does nothing in CE, because database load + # balancing is EE-only feature, but you can still use it in CE. It will + # start ensuring read consistency once it is overridden in EE. + # + # Using this class in CE helps to avoid creeping discrepancy between CE / + # EE only to force usage of the primary database in EE. + # + class Consistency + ## + # In CE there is no database load balancing, so all reads are expected to + # be consistent by the ACID guarantees of a single PostgreSQL instance. + # + # This method is overridden in EE. + # + def self.with_read_consistency(&block) + yield + end + end + end +end + +::Gitlab::Database::Consistency.singleton_class.prepend_if_ee('EE::Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb new file mode 100644 index 00000000000..f20a9b30fa7 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module V2 + include Gitlab::Database::MigrationHelpers + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + # Reverses operations performed by rename_column_concurrently. + # + # This method takes care of removing previously installed triggers as well + # as removing the new column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def undo_rename_column_concurrently(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: new_column) + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def cleanup_concurrent_column_rename(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: old_column) + end + + # Reverses the operations performed by cleanup_concurrent_column_rename. + # + # This method adds back the old_column removed + # by cleanup_concurrent_column_rename. + # It also adds back the triggers that are removed + # by cleanup_concurrent_column_rename. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the old column. If no type is given the new column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + # + def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + private + + def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name) + if transaction_open? + raise "#{calling_operation} can not be run inside a transaction" + end + + column = columns(table).find { |column| column.name == old_column.to_s } + + unless column + raise "Column #{old_column} does not exist on #{table}" + end + + if column.default + raise "#{calling_operation} does not currently support columns with default values" + end + + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + unless column_exists?(table, new_column) + create_column_from(table, old_column, new_column, type: type, batch_column_name: batch_column_name) + end + end + + def teardown_rename_mechanism(table, old_column, new_column, column_to_remove:) + return unless column_exists?(table, column_to_remove) + + with_lock_retries do + check_trigger_permissions!(table) + + remove_bidirectional_triggers(table, old_column, new_column) + + remove_column(table, column_to_remove) + end + end + + def install_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) + + create_insert_trigger(insert_trigger_name, quoted_table, quoted_old, quoted_new) + create_update_trigger(update_old_trigger_name, quoted_table, quoted_new, quoted_old) + create_update_trigger(update_new_trigger_name, quoted_table, quoted_old, quoted_new) + end + + def remove_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + + drop_trigger(insert_trigger_name, quoted_table) + drop_trigger(update_old_trigger_name, quoted_table) + drop_trigger(update_new_trigger_name, quoted_table) + end + + def bidirectional_trigger_names(table, old_column, new_column) + %w[insert update_old update_new].map do |operation| + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old_column}_#{new_column}_#{operation}").first(12) + end + end + + def function_name_for_trigger(trigger_name) + "function_for_#{trigger_name}" + end + + def create_insert_trigger(trigger_name, quoted_table, quoted_old_column, quoted_new_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.#{quoted_old_column} IS NULL AND NEW.#{quoted_new_column} IS NOT NULL THEN + NEW.#{quoted_old_column} = NEW.#{quoted_new_column}; + END IF; + + IF NEW.#{quoted_new_column} IS NULL AND NEW.#{quoted_old_column} IS NOT NULL THEN + NEW.#{quoted_new_column} = NEW.#{quoted_old_column}; + END IF; + + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE INSERT ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def create_update_trigger(trigger_name, quoted_table, quoted_source_column, quoted_target_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW.#{quoted_target_column} := NEW.#{quoted_source_column}; + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE UPDATE OF #{quoted_source_column} ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def drop_trigger(trigger_name, quoted_table) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + DROP FUNCTION IF EXISTS #{function_name}; + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 686dda80207..f4cf576dda7 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -164,8 +164,8 @@ module Gitlab "this could indicate the previous partitioning migration has been rolled back." end - Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |raw_arguments| - JobArguments.from_array(raw_arguments).source_table_name == table_name.to_s + Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |background_job| + JobArguments.from_array(background_job.args.second).source_table_name == table_name.to_s end primary_key = connection.primary_key(table_name) diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb new file mode 100644 index 00000000000..c8bb39e9f5d --- /dev/null +++ b/lib/gitlab/diff/char_diff.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class CharDiff + include Gitlab::Utils::StrongMemoize + + def initialize(old_string, new_string) + @old_string = old_string.to_s + @new_string = new_string.to_s + @changes = [] + end + + def generate_diff + @changes = diff_match_patch.diff_main(@old_string, @new_string) + diff_match_patch.diff_cleanupSemantic(@changes) + + @changes + end + + def changed_ranges(offset: 0) + old_diffs = [] + new_diffs = [] + new_pointer = old_pointer = offset + + generate_diff.each do |(action, content)| + content_size = content.size + + if action == :equal + new_pointer += content_size + old_pointer += content_size + end + + if action == :delete + old_diffs << (old_pointer..(old_pointer + content_size - 1)) + old_pointer += content_size + end + + if action == :insert + new_diffs << (new_pointer..(new_pointer + content_size - 1)) + new_pointer += content_size + end + end + + [old_diffs, new_diffs] + end + + def to_html + @changes.map do |op, text| + %{<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>} + end.join.html_safe + end + + private + + def diff_match_patch + strong_memoize(:diff_match_patch) { DiffMatchPatch.new } + end + + def html_class_names(operation) + class_names = ['idiff'] + + case operation + when :insert + class_names << 'addition' + when :delete + class_names << 'deletion' + end + + class_names.join(' ') + end + end + end +end diff --git a/lib/gitlab/diff/file_collection_sorter.rb b/lib/gitlab/diff/file_collection_sorter.rb index 94626875580..7b099543c83 100644 --- a/lib/gitlab/diff/file_collection_sorter.rb +++ b/lib/gitlab/diff/file_collection_sorter.rb @@ -3,6 +3,10 @@ module Gitlab module Diff class FileCollectionSorter + B_FOLLOWS_A = 1 + A_FOLLOWS_B = -1 + EQUIVALENT = 0 + attr_reader :diffs def initialize(diffs) @@ -29,14 +33,16 @@ module Gitlab a_part = a_parts.shift b_part = b_parts.shift - return 1 if a_parts.size < b_parts.size && a_parts.empty? - return -1 if a_parts.size > b_parts.size && b_parts.empty? + return B_FOLLOWS_A if a_parts.size < b_parts.size && a_parts.empty? + return A_FOLLOWS_B if a_parts.size > b_parts.size && b_parts.empty? comparison = a_part <=> b_part - return comparison unless comparison == 0 + return comparison unless comparison == EQUIVALENT + return compare_path_parts(a_parts, b_parts) if a_parts.any? && b_parts.any? - compare_path_parts(a_parts, b_parts) + # If A and B have the same name (e.g. symlink change), they are identical so return 0 + EQUIVALENT end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a5259079345..035084d4861 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,12 +3,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines, :repository + attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository + @project = repository&.project if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @@ -66,7 +67,7 @@ module Gitlab end def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines) + @inline_diffs ||= InlineDiff.for_lines(@raw_lines, project: project) end def old_lines diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 90cb9c8638a..5141b5170f0 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -8,6 +8,7 @@ module Gitlab EXPIRATION = 1.week VERSION = 1 + NEXT_VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection @@ -69,12 +70,20 @@ module Gitlab def key strong_memoize(:redis_key) do - ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":") + ['highlighted-diff-files', diffable.cache_key, version, diff_options].join(":") end end private + def version + if Feature.enabled?(:improved_merge_diff_highlighting, diffable.project) + NEXT_VERSION + else + VERSION + end + end + def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 5815d1bae4a..9b3fe1e3a43 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -27,28 +27,19 @@ module Gitlab @offset = offset end - def inline_diffs + def inline_diffs(project: nil) # Skip inline diff if empty line was replaced with content return if old_line == "" - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) - - lcp += offset - old_length = old_line.length + offset - new_length = new_line.length + offset - - old_diff_range = lcp..(old_length - lcs - 1) - new_diff_range = lcp..(new_length - lcs - 1) - - old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end - - [old_diffs, new_diffs] + if Feature.enabled?(:improved_merge_diff_highlighting, project) + CharDiff.new(old_line, new_line).changed_ranges(offset: offset) + else + deprecated_diff + end end class << self - def for_lines(lines) + def for_lines(lines, project: nil) changed_line_pairs = find_changed_line_pairs(lines) inline_diffs = [] @@ -57,7 +48,7 @@ module Gitlab old_line = lines[old_index] new_line = lines[new_index] - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs(project: project) inline_diffs[old_index] = old_diffs inline_diffs[new_index] = new_diffs @@ -97,6 +88,24 @@ module Gitlab private + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/299884 + def deprecated_diff + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 196203211ed..3cb4798a940 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -6,6 +6,7 @@ # Experiment options: # - tracking_category (optional, used to set the category when tracking an experiment event) # - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility -- you likely do not need this, see note in the next paragraph.) +# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout # # Using the backwards-compatible subject index (use_backwards_compatible_subject_index option): # This option was added when [the calculation of experimentation_subject_index was changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45733/diffs#41af4a6fa5a10c7068559ce21c5188483751d934_157_173). It is not intended to be used by new experiments, it exists merely for the segmentation integrity of in-flight experiments at the time the change was deployed. That is, we want users who were assigned to the "experimental" group or the "control" group before the change to still be in those same groups after the change. See [the original issue](https://gitlab.com/gitlab-org/gitlab/-/issues/270858) and [this related comment](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48110#note_458223745) for more information. @@ -69,10 +70,6 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', use_backwards_compatible_subject_index: true }, - default_to_issues_board: { - tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard', - use_backwards_compatible_subject_index: true - }, jobs_empty_state: { tracking_category: 'Growth::Activation::Experiment::JobsEmptyState' }, @@ -92,19 +89,25 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, ci_syntax_templates: { - tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates' + tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', + rollout_strategy: :user }, pipelines_empty_state: { - tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState' + tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState', + rollout_strategy: :user }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, show_trial_status_in_sidebar: { - tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar' + tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', + rollout_strategy: :group }, trial_onboarding_issues: { tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' + }, + in_product_marketing_emails: { + tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' } }.freeze @@ -126,12 +129,44 @@ module Gitlab return false if subject.blank? return false unless active?(experiment_key) + log_invalid_rollout(experiment_key, subject) + experiment = get_experiment(experiment_key) return false unless experiment experiment.enabled_for_index?(index_for_subject(experiment, subject)) end + def rollout_strategy(experiment_key) + experiment = get_experiment(experiment_key) + return unless experiment + + experiment.rollout_strategy + end + + def log_invalid_rollout(experiment_key, subject) + return if valid_subject_for_rollout_strategy?(experiment_key, subject) + + logger = Gitlab::ExperimentationLogger.build + logger.warn message: 'Subject must conform to the rollout strategy', + experiment_key: experiment_key, + subject: subject.class.to_s, + rollout_strategy: rollout_strategy(experiment_key) + end + + def valid_subject_for_rollout_strategy?(experiment_key, subject) + case rollout_strategy(experiment_key) + when :user + subject.is_a?(User) + when :group + subject.is_a?(Group) + when :cookie + subject.nil? || subject.is_a?(String) + else + false + end + end + private def index_for_subject(experiment, subject) diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index e43f3c8c007..2b38b12c914 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -40,6 +40,8 @@ module Gitlab return true if forced_enabled?(experiment_key) return false if dnt_enabled? + Experimentation.log_invalid_rollout(experiment_key, subject) + subject ||= fallback_experimentation_subject_index(experiment_key) Experimentation.in_experiment_group?(experiment_key, subject: subject) @@ -65,7 +67,9 @@ module Gitlab return if dnt_enabled? return unless Experimentation.active?(experiment_key) && current_user - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: current_user), current_user, context) + subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) end def record_experiment_conversion_event(experiment_key) @@ -136,7 +140,7 @@ module Gitlab cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } end - def tracking_label(subject) + def tracking_label(subject = nil) return experimentation_subject_id if subject.blank? if subject.respond_to?(:to_global_id) diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 36cd673a38f..17dda45f5b7 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -5,12 +5,13 @@ module Gitlab class Experiment FEATURE_FLAG_SUFFIX = "_experiment_percentage" - attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index + attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index, :rollout_strategy def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] + @rollout_strategy = params[:rollout_strategy] || :cookie end def active? diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb new file mode 100644 index 00000000000..ba1b60d6b4c --- /dev/null +++ b/lib/gitlab/experimentation_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class ExperimentationLogger < ::Gitlab::JsonLogger + def self.file_name_noext + 'experimentation_json' + end + end +end diff --git a/lib/gitlab/faraday.rb b/lib/gitlab/faraday.rb deleted file mode 100644 index f92392ec1a9..00000000000 --- a/lib/gitlab/faraday.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Faraday - ::Faraday::Request.register_middleware(gitlab_error_callback: -> { ::Gitlab::Faraday::ErrorCallback }) - end -end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index 38ccd2c38a9..ed25310b5cf 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -19,7 +19,7 @@ # `Content-Type` and `Content-Disposition` to the one we get from the detection. module Gitlab module FileTypeDetection - SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico webp].freeze SAFE_IMAGE_FOR_SCALING_EXT = %w[png jpg jpeg].freeze PDF_EXT = 'pdf' diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 0bc7ecccf5e..35c3dc5b0b3 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -16,7 +16,7 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email + :committed_date, :committer_name, :committer_email, :trailers ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -389,6 +389,7 @@ module Gitlab @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) + @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }] end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 209917073c7..53df0b7b389 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -244,6 +244,8 @@ module Gitlab def prune_diff_if_eligible if too_large? + ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) + too_large! elsif collapsed? collapse! diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 0eff35ab1c4..0607b151de2 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -103,6 +103,7 @@ module Gitlab @committer_name = committer[:name] @committer_email = committer[:email] @parent_ids = commit.parents.map(&:oid) + @trailers = Hash[commit.trailers] end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 31734abe77f..c51349b9113 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -203,7 +203,7 @@ module Gitlab def self.authorization_token(storage) token = token(storage).to_s issued_at = real_time.to_i.to_s - hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('SHA256'), token, issued_at) "v2.#{hmac}.#{issued_at}" end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ea940150941..ef5221a8042 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -335,7 +335,8 @@ module Gitlab all: !!options[:all], first_parent: !!options[:first_parent], global_options: parse_global_options!(options), - disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. + disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. + trailers: options[:trailers] ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/global_id.rb b/lib/gitlab/global_id.rb index e8a6006dce1..7e9412236cf 100644 --- a/lib/gitlab/global_id.rb +++ b/lib/gitlab/global_id.rb @@ -19,8 +19,8 @@ module Gitlab value when URI::GID GlobalID.new(value) - when Integer - raise CoerceError, 'Cannot coerce Integer' unless model_name.present? + when Integer, String + raise CoerceError, "Cannot coerce #{value.class}" unless model_name.present? GlobalID.new(::Gitlab::GlobalId.build(model_name: model_name, id: value)) else diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 0ba535b500e..46c7935cb1d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -43,7 +43,6 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: true) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 54a84be4274..965c01dd02f 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -6,6 +6,10 @@ module Gitlab module Connections def self.use(schema) schema.connections.add( + ::Gitlab::Graphql::Pagination::OffsetPaginatedRelation, + ::Gitlab::Graphql::Pagination::OffsetActiveRecordRelationConnection) + + schema.connections.add( ActiveRecord::Relation, Gitlab::Graphql::Pagination::Keyset::Connection) diff --git a/lib/gitlab/graphql/pagination/offset_paginated_relation.rb b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb new file mode 100644 index 00000000000..8a8c6e5db50 --- /dev/null +++ b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Marker class to enable us to choose the correct +# connection type during resolution +module Gitlab + module Graphql + module Pagination + class OffsetPaginatedRelation < SimpleDelegator + end + end + end +end diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb index 434d30d9717..e5bae61ae4e 100644 --- a/lib/gitlab/hook_data/base_builder.rb +++ b/lib/gitlab/hook_data/base_builder.rb @@ -21,6 +21,12 @@ module Gitlab private + def event_data(event) + event_name = "#{object.class.name.downcase}_#{event}" + + { event_name: event_name } + end + def timestamps_data { created_at: object.created_at&.xmlschema, diff --git a/lib/gitlab/hook_data/group_builder.rb b/lib/gitlab/hook_data/group_builder.rb new file mode 100644 index 00000000000..5f76144eb83 --- /dev/null +++ b/lib/gitlab/hook_data/group_builder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class GroupBuilder < BaseBuilder + alias_method :group, :object + + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"group_rename", + # :name=>"group1", + # :path=>"group1", + # :full_path=>"group1", + # :group_id=>1, + # :old_path=>"old-path", + # :old_full_path=>"old-path" + # } + + def build(event) + [ + timestamps_data, + event_data(event), + group_data, + event_specific_group_data(event) + ].reduce(:merge) + end + + private + + def group_data + { + name: group.name, + path: group.path, + full_path: group.full_path, + group_id: group.id + } + end + + def event_specific_group_data(event) + return {} unless event == :rename + + { + old_path: group.path_before_last_save, + old_full_path: group.full_path_before_last_save + } + end + end + end +end diff --git a/lib/gitlab/hook_data/subgroup_builder.rb b/lib/gitlab/hook_data/subgroup_builder.rb new file mode 100644 index 00000000000..a620219675a --- /dev/null +++ b/lib/gitlab/hook_data/subgroup_builder.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class SubgroupBuilder < GroupBuilder + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"subgroup_create", + # :name=>"subgroup1", + # :path=>"subgroup1", + # :full_path=>"group1/subgroup1", + # :group_id=>10, + # :parent_group_id=>7, + # :parent_name=>group1, + # :parent_path=>group1, + # :parent_full_path=>group1 + # } + + private + + def event_data(event) + event_name = case event + when :create + 'subgroup_create' + when :destroy + 'subgroup_destroy' + end + + { event_name: event_name } + end + + def group_data + parent = group.parent + + super.merge( + parent_group_id: parent.id, + parent_name: parent.name, + parent_path: parent.path, + parent_full_path: parent.full_path + ) + end + + def event_specific_group_data(event) + {} + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 921072a4970..c4867746b0f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -103,3 +103,7 @@ module Gitlab end Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') + +# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both +# instance and class methods. +Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper') diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb index a702c58a7c2..e093b4b0697 100644 --- a/lib/gitlab/import_export/design_repo_restorer.rb +++ b/lib/gitlab/import_export/design_repo_restorer.rb @@ -3,10 +3,11 @@ module Gitlab module ImportExport class DesignRepoRestorer < RepoRestorer - def initialize(project:, shared:, path_to_bundle:) - super(project: project, shared: shared, path_to_bundle: path_to_bundle) + extend ::Gitlab::Utils::Override - @repository = project.design_repository + override :repository + def repository + @repository ||= importable.design_repository end # `restore` method is handled in super class diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb index db9ebee6a13..b400aedc205 100644 --- a/lib/gitlab/import_export/design_repo_saver.rb +++ b/lib/gitlab/import_export/design_repo_saver.rb @@ -3,16 +3,18 @@ module Gitlab module ImportExport class DesignRepoSaver < RepoSaver - def save - @repository = project.design_repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.design_repository end private - def bundle_full_path - File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.design_repo_bundle_filename end end end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index dfe27118d66..925ab6680ba 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -6,7 +6,7 @@ module Gitlab class TreeRestorer include Gitlab::Utils::StrongMemoize - attr_reader :user, :shared + attr_reader :user, :shared, :groups_mapping def initialize(user:, shared:, group:) @user = user diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 789249c7d91..390909efe36 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -75,19 +75,19 @@ module Gitlab def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: shared, - project: project) + importable: project) end def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: shared, - project: ProjectWiki.new(project)) + importable: ProjectWiki.new(project)) end def design_repo_restorer Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path, shared: shared, - project: project) + importable: project) end def uploads_restorer diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index f808e30bd6e..7701916a855 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -5,10 +5,12 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) - @repository = project.repository + attr_reader :importable + + def initialize(importable:, shared:, path_to_bundle:) @path_to_bundle = path_to_bundle @shared = shared + @importable = importable end def restore @@ -22,9 +24,13 @@ module Gitlab false end + def repository + @repository ||= importable.repository + end + private - attr_accessor :repository, :path_to_bundle, :shared + attr_accessor :path_to_bundle, :shared def ensure_repository_does_not_exist! if repository.exists? diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 898cd7898ba..0fdd0722b65 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,12 +5,11 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :project, :repository, :shared + attr_reader :exportable, :shared - def initialize(project:, shared:) - @project = project + def initialize(exportable:, shared:) + @exportable = exportable @shared = shared - @repository = @project.repository end def save @@ -19,6 +18,10 @@ module Gitlab bundle_to_disk end + def repository + @repository ||= @exportable.repository + end + private def repository_exists? @@ -26,11 +29,16 @@ module Gitlab end def bundle_full_path - File.join(shared.export_path, ImportExport.project_bundle_filename) + File.join(shared.export_path, bundle_filename) + end + + def bundle_filename + ::Gitlab::ImportExport.project_bundle_filename end def bundle_to_disk - mkdir_p(shared.export_path) + mkdir_p(File.dirname(bundle_full_path)) + repository.bundle_to_disk(bundle_full_path) rescue => e shared.error(e) diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 045ba2495bf..bb2bbda4bd6 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -31,7 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_base_tmp_dir + remove_archive_tmp_dir end private @@ -40,8 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_base_tmp_dir - FileUtils.rm_rf(@shared.base_path) + def remove_archive_tmp_dir + FileUtils.rm_rf(@shared.archive_path) end def archive_file diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 93ae6f6b02a..4b1cf4915e4 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -3,18 +3,21 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver - def save - wiki = ProjectWiki.new(project) - @repository = wiki.repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.wiki.repository end private - def bundle_full_path - File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.wiki_repo_bundle_filename end end end end + +Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 6800e5667f6..644a5fc4fff 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -61,7 +61,7 @@ module Gitlab key_slot(args.first) end - unless key_slots.uniq.length == 1 + if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord raise CrossSlotError.new("Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 6b0f01757b7..1c7a2056c21 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -14,7 +14,9 @@ module Gitlab :elasticsearch_calls, :elasticsearch_duration_s, *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS] + *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS] end def add_instrumentation_data(payload) @@ -24,6 +26,8 @@ module Gitlab instrument_elasticsearch(payload) instrument_throttle(payload) instrument_active_record(payload) + instrument_external_http(payload) + instrument_rack_attack(payload) end def instrument_gitaly(payload) @@ -59,6 +63,14 @@ module Gitlab payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time end + def instrument_external_http(payload) + external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count + + return if external_http_count == 0 + + payload.merge! Gitlab::Metrics::Subscribers::ExternalHttp.payload + end + def instrument_throttle(payload) safelist = Gitlab::Instrumentation::Throttle.safelist payload[:throttle_safelist] = safelist if safelist.present? @@ -70,6 +82,13 @@ module Gitlab payload.merge!(db_counters) end + def instrument_rack_attack(payload) + rack_attack_redis_count = ::Gitlab::Metrics::Subscribers::RackAttack.payload[:rack_attack_redis_count] + return if rack_attack_redis_count == 0 + + payload.merge!(::Gitlab::Metrics::Subscribers::RackAttack.payload) + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 08dde98e965..329c0f221b5 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -23,6 +23,12 @@ module Gitlab write_secret end + + def included_in_gitlab_com_rollout?(project) + return true unless ::Gitlab.com? + + Feature.enabled?(:kubernetes_agent_on_gitlab_com, project) + end end end end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb index f603ff44ef3..17ea2eb5188 100644 --- a/lib/gitlab/kubernetes/helm/v2/certificate.rb +++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb @@ -59,7 +59,7 @@ module Gitlab cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) end - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + cert.sign(signed_by&.key || key, OpenSSL::Digest.new('SHA256')) new(key, cert) end diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb new file mode 100644 index 00000000000..94c5d965200 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in external HTTP + # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 + class ExternalHttp < ActiveSupport::Subscriber + attach_to :external_http + + DEFAULT_STATUS_CODE = 'undefined' + + DETAIL_STORE = :external_http_detail_store + COUNTER = :external_http_count + DURATION = :external_http_duration_s + + KNOWN_PAYLOAD_KEYS = [COUNTER, DURATION].freeze + + def self.detail_store + ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] + end + + def self.duration + Gitlab::SafeRequestStore[DURATION].to_f + end + + def self.request_count + Gitlab::SafeRequestStore[COUNTER].to_i + end + + def self.payload + { + COUNTER => request_count, + DURATION => duration + } + end + + def request(event) + payload = event.payload + add_to_detail_store(payload) + add_to_request_store(payload) + expose_metrics(payload) + end + + private + + def current_transaction + ::Gitlab::Metrics::Transaction.current + end + + def add_to_detail_store(payload) + return unless Gitlab::PerformanceBar.enabled_for_request? + + self.class.detail_store << { + duration: payload[:duration], + scheme: payload[:scheme], + method: payload[:method], + host: payload[:host], + port: payload[:port], + path: payload[:path], + query: payload[:query], + code: payload[:code], + exception_object: payload[:exception_object], + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + + def add_to_request_store(payload) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + end + + def expose_metrics(payload) + return unless current_transaction + + labels = { method: payload[:method], code: payload[:code] || DEFAULT_STATUS_CODE } + + current_transaction.increment(:gitlab_external_http_total, 1, labels) do + docstring 'External HTTP calls' + label_keys labels.keys + end + + current_transaction.observe(:gitlab_external_http_duration_seconds, payload[:duration]) do + docstring 'External HTTP time' + buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] + end + + if payload[:exception_object].present? + current_transaction.increment(:gitlab_external_http_exception_total, 1) do + docstring 'External HTTP exceptions' + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb new file mode 100644 index 00000000000..2791a39fb16 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # - Adds logging for all Rack Attack blocks and throttling events. + # - Instrument the cache operations of RackAttack to use in structured + # logs. Two fields are exposed: + # + rack_attack_redis_count: the number of redis calls triggered by + # RackAttack in a request. + # + rack_attack_redis_duration_s: the total duration of all redis calls + # triggered by RackAttack in a request. + class RackAttack < ActiveSupport::Subscriber + attach_to 'rack_attack' + + INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation + + THROTTLES_WITH_USER_INFORMATION = [ + :throttle_authenticated_api, + :throttle_authenticated_web, + :throttle_authenticated_protected_paths_api, + :throttle_authenticated_protected_paths_web + ].freeze + + PAYLOAD_KEYS = [ + :rack_attack_redis_count, + :rack_attack_redis_duration_s + ].freeze + + def self.payload + Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { + rack_attack_redis_count: 0, + rack_attack_redis_duration_s: 0.0 + } + end + + def redis(event) + self.class.payload[:rack_attack_redis_count] += 1 + self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000 + end + + def safelist(event) + req = event.payload[:request] + Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched'] + end + + def throttle(event) + log_into_auth_logger(event) + end + + def blocklist(event) + log_into_auth_logger(event) + end + + def track(event) + log_into_auth_logger(event) + end + + private + + def log_into_auth_logger(event) + req = event.payload[:request] + rack_attack_info = { + message: 'Rack_Attack', + env: req.env['rack.attack.match_type'], + remote_ip: req.ip, + request_method: req.request_method, + path: req.fullpath, + matched: req.env['rack.attack.matched'] + } + + if THROTTLES_WITH_USER_INFORMATION.include? req.env['rack.attack.matched'].to_sym + user_id = req.env['rack.attack.match_discriminator'] + user = User.find_by(id: user_id) # rubocop:disable CodeReuse/ActiveRecord + + rack_attack_info[:user_id] = user_id + rack_attack_info['meta.user'] = user.username unless user.nil? + end + + Gitlab::InstrumentationHelper.add_instrumentation_data(rack_attack_info) + + logger.error(rack_attack_info) + end + + def logger + Gitlab::AuthLogger + end + end + end + end +end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index 22ece0a6a8b..dde78cd9178 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -39,9 +39,14 @@ module Gitlab def class_methods super + class_methods_module = const_get(:ClassMethods, false) + if instance_variable_defined?(:@_prepended_class_methods) - const_get(:ClassMethods, false).prepend @_prepended_class_methods + class_methods_module.prepend @_prepended_class_methods end + + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + extend class_methods_module if ENV['STATIC_VERIFICATION'] end def prepended(base = nil, &block) diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index d1504d88315..380340b80be 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -27,27 +27,40 @@ module Gitlab end def log_sql_queries(id, data) - return [] unless queries = data.dig('data', 'active-record', 'details') - - queries.each do |query| - next unless location = parse_backtrace(query['backtrace']) + queries_by_location(data).each do |location, queries| + next unless location - log_info = location.merge( + duration = queries.sum { |query| query['duration'].to_f } + log_info = { + method_path: "#{location[:filename]}:#{location[:method]}", + filename: location[:filename], type: :sql, request_id: id, - duration_ms: query['duration'].to_f - ) + count: queries.count, + duration_ms: duration + } logger.info(log_info) end end + def queries_by_location(data) + return [] unless queries = data.dig('data', 'active-record', 'details') + + queries.group_by do |query| + parse_backtrace(query['backtrace']) + end + end + def parse_backtrace(backtrace) return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first) { filename: match[:filename], - filenum: match[:filenum].to_i, + # filenum may change quite frequently with every change in the file, + # because the intention is to aggregate these queries, we group + # them rather by method name which should not change so frequently + # filenum: match[:filenum].to_i, method: match[:method] } end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index b56fd8278a1..90745dde0af 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -187,7 +187,7 @@ module Gitlab parse_params do |reviewer_param| extract_users(reviewer_param) end - command :assign_reviewer, :reviewer do |users| + command :assign_reviewer, :reviewer, :request_review do |users| next if users.empty? if quick_action_target.allows_multiple_reviewers? diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 2a94fb91880..ae3c89c3565 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -12,13 +12,15 @@ module Gitlab rack_attack::Request.include(Gitlab::RackAttack::Request) # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response - Rack::Attack.throttled_response = lambda do |env| + rack_attack.throttled_response = lambda do |env| throttled_headers = Gitlab::RackAttack.throttled_response_headers( env['rack.attack.matched'], env['rack.attack.match_data'] ) [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end + rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new + # Configure the throttles configure_throttles(rack_attack) diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb new file mode 100644 index 00000000000..8cf9082384f --- /dev/null +++ b/lib/gitlab/rack_attack/instrumented_cache_store.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + # This class is a proxy for all Redis calls made by RackAttack. All the + # calls are instrumented, then redirected to ::Rails.cache. This class + # instruments the standard interfaces of ActiveRecord::Cache defined in + # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 + # + # For more information, please see + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751 + class InstrumentedCacheStore + NOTIFICATION_CHANNEL = 'redis.rack_attack' + + delegate :silence!, :mute, to: :@upstream_store + + def initialize(upstream_store: ::Rails.cache, notifier: ActiveSupport::Notifications) + @upstream_store = upstream_store + @notifier = notifier + end + + [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete, + :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface| + define_method interface do |*args, **k_args, &block| + @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do + @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index f3cbe1db901..a08cea5a435 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -2,8 +2,10 @@ module Gitlab module Recaptcha + extend Gitlab::Utils::StrongMemoize + def self.load_configurations! - if Gitlab::CurrentSettings.recaptcha_enabled || enabled_on_login? + if enabled? || enabled_on_login? ::Recaptcha.configure do |config| config.site_key = Gitlab::CurrentSettings.recaptcha_site_key config.secret_key = Gitlab::CurrentSettings.recaptcha_private_key diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 5b1f9400bc7..c0420126ada 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -5,6 +5,9 @@ module Gitlab class Query < SimpleDelegator include EncodingHelper + QUOTES_REGEXP = %r{\A"|"\Z}.freeze + TOKEN_WITH_QUOTES_REGEXP = %r{\s(?=(?:[^"]|"[^"]*")*$)}.freeze + def initialize(query, filter_opts = {}, &block) @raw_query = query.dup @filters = [] @@ -35,22 +38,24 @@ module Gitlab def extract_filters fragments = [] + query_tokens = parse_raw_query filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ } + match = query_tokens.find { |part| part =~ /\A-?#{filter[:name]}:/ } + next unless match input = match.split(':')[1..-1].join next if input.empty? filter[:negated] = match.start_with?("-") - filter[:value] = parse_filter(filter, input) + filter[:value] = parse_filter(filter, input.gsub(QUOTES_REGEXP, '')) filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') fragments << match parsed_filters << filter end - query = (@raw_query.split - fragments).join(' ') + query = (query_tokens - fragments).join(' ') query = '*' if query.empty? [query, filters] @@ -61,6 +66,13 @@ module Gitlab @filter_options[:encode_binary] ? encode_binary(result) : result end + + def parse_raw_query + # Positive lookahead for any non-quote char or even number of quotes + # for example '"search term" path:"foo bar.txt"' would break into + # ["search term", "path:\"foo bar.txt\""] + @raw_query.split(TOKEN_WITH_QUOTES_REGEXP).reject(&:empty?) + end end end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index eb845c5ff8d..f7b826ba648 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -13,7 +13,7 @@ module Gitlab base_payload = parse_job(job) ActiveRecord::LogSubscriber.reset_runtime - Sidekiq.logger.info log_job_start(base_payload) + Sidekiq.logger.info log_job_start(job, base_payload) yield @@ -40,13 +40,15 @@ module Gitlab output_payload.merge!(job.slice(*::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS)) end - def log_job_start(payload) + def log_job_start(job, payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s + payload['job_size_bytes'] = Sidekiq.dump_json(job).bytesize + payload end diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb index d59a8fc3730..5bca3efe6e1 100644 --- a/lib/gitlab/suggestions/commit_message.rb +++ b/lib/gitlab/suggestions/commit_message.rb @@ -6,14 +6,15 @@ module Gitlab DEFAULT_SUGGESTION_COMMIT_MESSAGE = 'Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)' - def initialize(user, suggestion_set) + def initialize(user, suggestion_set, custom_message = nil) @user = user @suggestion_set = suggestion_set + @custom_message = custom_message end def message project = suggestion_set.project - user_defined_message = project.suggestion_commit_message.presence + user_defined_message = @custom_message.presence || project.suggestion_commit_message.presence message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE Gitlab::StringPlaceholderReplacer diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index c702c6f1add..db3c058184c 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -66,6 +66,18 @@ module Gitlab answer end + # Prompt the user to input a password + # + # message - custom message to display before input + def prompt_for_password(message = 'Enter password: ') + unless STDIN.tty? + print(message) + return STDIN.gets.chomp + end + + STDIN.getpass(message) + end + # Runs the given command and matches the output against the given pattern # # Returns nil if nothing matched diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 9b39d386674..6d2677175e6 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -5,9 +5,10 @@ module Gitlab module Template module Finders class GlobalTemplateFinder < BaseTemplateFinder - def initialize(base_dir, extension, categories = {}, excluded_patterns: []) + def initialize(base_dir, extension, categories = {}, include_categories_for_file = {}, excluded_patterns: []) @categories = categories @extension = extension + @include_categories_for_file = include_categories_for_file @excluded_patterns = excluded_patterns super(base_dir) @@ -47,7 +48,9 @@ module Gitlab end def select_directory(file_name) - @categories.keys.find do |category| + categories = @categories + categories.merge!(@include_categories_for_file[file_name]) if @include_categories_for_file[file_name].present? + categories.keys.find do |category| File.exist?(File.join(category_directory(category), file_name)) end end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index c295cc75da5..01158cafc4f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -25,6 +25,12 @@ module Gitlab } end + def include_categories_for_file + { + "SAST#{self.extension}" => { 'Security' => 'Security' } + } + end + def excluded_patterns strong_memoize(:excluded_patterns) do BASE_EXCLUDED_PATTERNS + additional_excluded_patterns @@ -41,7 +47,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories, excluded_patterns: self.excluded_patterns + self.base_dir, + self.extension, + self.categories, + self.include_categories_for_file, + excluded_patterns: self.excluded_patterns ) end end diff --git a/lib/gitlab/terraform/state_migration_helper.rb b/lib/gitlab/terraform/state_migration_helper.rb new file mode 100644 index 00000000000..04c1cbd0373 --- /dev/null +++ b/lib/gitlab/terraform/state_migration_helper.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Terraform + class StateMigrationHelper + class << self + def migrate_to_remote_storage(&block) + migrate_in_batches( + ::Terraform::StateVersion.with_files_stored_locally.preload_state, + ::Terraform::StateUploader::Store::REMOTE, + &block + ) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate_in_batches(versions, store, &block) + versions.find_each(batch_size: batch_size) do |version| # rubocop:disable CodeReuse/ActiveRecord + version.file.migrate!(store) + + yield version if block_given? + end + end + end + end + end +end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 71dfe27dd5a..0c4911ba47e 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,32 +3,31 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-1'.freeze + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze + GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze - def initialize(namespace: nil, project: nil, **data) - @namespace = namespace - @project = project + def initialize(namespace: nil, project: nil, user: nil, **data) @data = data end - def namespace_id - namespace&.id + def to_context + SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) end - def project_id - @project&.id + def environment + return 'production' if Gitlab.com_and_canary? + + return 'staging' if Gitlab.staging? + + 'development' end - def to_context - SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) + def source + GITLAB_RAILS_SOURCE end private - def namespace - @namespace || @project&.namespace - end - def to_h public_methods(false).each_with_object({}) do |method, hash| next if method == :to_context diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb new file mode 100644 index 00000000000..aa778f7f26f --- /dev/null +++ b/lib/gitlab/usage/docs/helper.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + # Helper with functions to be used by HAML templates + module Helper + HEADER = %w(field value).freeze + SKIP_KEYS = %i(description).freeze + + def auto_generated_comment + <<-MARKDOWN.strip_heredoc + --- + stage: Growth + group: Product Intelligence + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + --- + + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. + ---> + + <!-- vale gitlab.Spelling = NO --> + MARKDOWN + end + + def render_name(name) + "## #{name}\n" + end + + def render_description(object) + object.description + end + + def render_attribute_row(key, value) + value = Gitlab::Usage::Docs::ValueFormatter.format(key, value) + table_row(["`#{key}`", value]) + end + + def render_attributes_table(object) + <<~MARKDOWN + + #{table_row(HEADER)} + #{table_row(HEADER.map { '---' })} + #{table_value_rows(object.attributes)} + MARKDOWN + end + + def table_value_rows(attributes) + attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value| + render_attribute_row(key, value) + end.join("\n") + end + + def table_row(array) + "| #{array.join(' | ')} |" + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb new file mode 100644 index 00000000000..7a7c58005bb --- /dev/null +++ b/lib/gitlab/usage/docs/renderer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class Renderer + include Gitlab::Usage::Docs::Helper + DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping') + TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml') + + def initialize(metrics_definitions) + @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) + @metrics_definitions = metrics_definitions.sort + end + + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '') + end + + def write + filename = DICTIONARY_PATH.join('dictionary.md').to_s + + FileUtils.mkdir_p(DICTIONARY_PATH) + File.write(filename, contents) + + filename + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml new file mode 100644 index 00000000000..86e93be66c7 --- /dev/null +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -0,0 +1,28 @@ += auto_generated_comment + +:plain + # Metrics Dictionary + + This file is autogenerated, please do not edit directly. + + To generate these files from the GitLab repository, run: + + ```shell + bundle exec rake gitlab:usage_data:generate_metrics_dictionary + ``` + + The Metrics Dictionary is based on the following metrics definition YAML files: + + - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics') + - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) + +Each table includes a `milestone`, which corresponds to the GitLab version when the metric +was released. +\ +- metrics_definitions.each do |name, object| + + = render_name(name) + + = render_description(object) + + = render_attributes_table(object) diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb new file mode 100644 index 00000000000..37db377ccba --- /dev/null +++ b/lib/gitlab/usage/docs/value_formatter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class ValueFormatter + def self.format(key, value) + case key + when :key_path + "**#{value}**" + when :data_source + value.capitalize + when :group + "`#{value}`" + when :introduced_by_url + "[Introduced by](#{value})" + when :distribution, :tier + Array(value).join(', ') + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index e1648c78168..f3469209f48 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -7,16 +7,16 @@ module Gitlab InvalidMetricError = Class.new(RuntimeError) - attr_accessor :default_generation_path, :value + attr_accessor :key_path, :value - validates :default_generation_path, presence: true + validates :key_path, presence: true def definition - self.class.definitions[default_generation_path] + self.class.definitions[key_path] end - def unflatten_default_path - unflatten(default_generation_path.split('.'), value) + def unflatten_key_path + unflatten(key_path.split('.'), value) end class << self diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 96e572bb3db..01d202e4d45 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -13,9 +13,8 @@ module Gitlab @attributes = opts end - # The key is defined by default_generation and full_path def key - full_path[default_generation.to_sym] + key_path end def to_h @@ -23,8 +22,10 @@ module Gitlab end def validate! - self.class.schemer.validate(attributes.stringify_keys).map do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + unless skip_validation? + self.class.schemer.validate(attributes.stringify_keys).each do |error| + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + end end end @@ -79,6 +80,10 @@ module Gitlab def method_missing(method, *args) attributes[method] || super end + + def skip_validation? + !!attributes[:skip_validation] + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb new file mode 100644 index 00000000000..72f14448656 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml') + UnknownAggregationOperator = Class.new(StandardError) + + class Aggregate + delegate :calculate_events_union, + :weekly_time_range, + :monthly_time_range, + to: Gitlab::UsageDataCounters::HLLRedisCounter + + def initialize + @aggregated_metrics = load_events(AGGREGATED_METRICS_PATH) + end + + def monthly_data + aggregated_metrics_data(**monthly_time_range) + end + + def weekly_data + aggregated_metrics_data(**weekly_time_range) + end + + private + + attr_accessor :aggregated_metrics + + def aggregated_metrics_data(start_date:, end_date:) + aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) + + weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: end_date) + end + end + + def calculate_count_for_aggregation(aggregation, start_date:, end_date:) + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) + Gitlab::Utils::UsageData::FALLBACK + end + rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::Utils::UsageData::FALLBACK + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_events = begin + subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ + calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1..(event_names.size - 1)) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + event_names.combination(subset_size).sum do |events_subset| + subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ + calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + event_names.sum do |event| + subset_powers_cache[subset_size][event] ||= \ + calculate_events_union(event_names: event, start_date: start_date, end_date: end_date) + end + end + end + end + + def load_events(wildcard) + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f935c677930..10566784975 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -12,6 +12,8 @@ # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } module Gitlab class UsageData + DEPRECATED_VALUE = -1000 + CE_MEMOIZED_VALUES = %i( issue_minimum_id issue_maximum_id @@ -23,6 +25,8 @@ module Gitlab deployment_minimum_id deployment_maximum_id auth_providers + aggregated_metrics + recorded_at ).freeze class << self @@ -75,7 +79,7 @@ module Gitlab end def recorded_at - Time.current + @recorded_at ||= Time.current end # rubocop: disable Metrics/AbcSize @@ -580,27 +584,35 @@ module Gitlab users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), + unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { - gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id) + gitlab: DEPRECATED_VALUE, + gitlab_v1: count(::BulkImport.where(time_period, source_type: :gitlab)) }, + project_imports: project_imports(time_period), + issue_imports: issue_imports(time_period), + group_imports: group_imports(time_period), + + # Deprecated data to be removed projects_imported: { - total: distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id), - gitlab_project: projects_imported_count('gitlab_project', time_period), - gitlab: projects_imported_count('gitlab', time_period), - github: projects_imported_count('github', time_period), - bitbucket: projects_imported_count('bitbucket', time_period), - bitbucket_server: projects_imported_count('bitbucket_server', time_period), - gitea: projects_imported_count('gitea', time_period), - git: projects_imported_count('git', time_period), - manifest: projects_imported_count('manifest', time_period) + total: DEPRECATED_VALUE, + gitlab_project: DEPRECATED_VALUE, + gitlab: DEPRECATED_VALUE, + github: DEPRECATED_VALUE, + bitbucket: DEPRECATED_VALUE, + bitbucket_server: DEPRECATED_VALUE, + gitea: DEPRECATED_VALUE, + git: DEPRECATED_VALUE, + manifest: DEPRECATED_VALUE }, issues_imported: { - jira: distinct_count(::JiraImportState.where(time_period), :user_id), - fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period), - csv: distinct_count(Issues::CsvImport.where(time_period), :user_id) + jira: DEPRECATED_VALUE, + fogbugz: DEPRECATED_VALUE, + phabricator: DEPRECATED_VALUE, + csv: DEPRECATED_VALUE }, - groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) + groups_imported: DEPRECATED_VALUE + # End of deprecated keys } end # rubocop: enable CodeReuse/ActiveRecord @@ -690,13 +702,13 @@ module Gitlab def aggregated_metrics_monthly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data + aggregated_metrics: aggregated_metrics.monthly_data } end def aggregated_metrics_weekly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data + aggregated_metrics: aggregated_metrics.weekly_data } end @@ -741,6 +753,10 @@ module Gitlab private + def aggregated_metrics + @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new + end + def event_monthly_active_users(date_range) data = { action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, @@ -893,10 +909,52 @@ module Gitlab count relation, start: deployment_minimum_id, finish: deployment_maximum_id end + def project_imports(time_period) + { + gitlab_project: projects_imported_count('gitlab_project', time_period), + gitlab: projects_imported_count('gitlab', time_period), + github: projects_imported_count('github', time_period), + bitbucket: projects_imported_count('bitbucket', time_period), + bitbucket_server: projects_imported_count('bitbucket_server', time_period), + gitea: projects_imported_count('gitea', time_period), + git: projects_imported_count('git', time_period), + manifest: projects_imported_count('manifest', time_period), + gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + def projects_imported_count(from, time_period) - distinct_count(::Project.imported_from(from).where(time_period).where.not(import_type: nil), :creator_id) # rubocop: disable CodeReuse/ActiveRecord + count(::Project.imported_from(from).where(time_period).where.not(import_type: nil)) # rubocop: disable CodeReuse/ActiveRecord end + def issue_imports(time_period) + { + jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + fogbugz: projects_imported_count('fogbugz', time_period), + phabricator: projects_imported_count('phabricator', time_period), + csv: count(Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord + } + end + + def group_imports(time_period) + { + group_import: count(::GroupImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + gitlab_migration: count(::BulkImports::Entity.where(time_period).group_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + + # rubocop:disable CodeReuse/ActiveRecord + def unique_users_all_imports(time_period) + project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) + bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) + jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) + csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) + group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) + + project_imports + bulk_imports + jira_issue_imports + csv_issue_imports + group_imports + end + # rubocop:enable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 47361d831b2..ed2ce2cecb0 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -13,15 +13,10 @@ module Gitlab AggregationMismatch = Class.new(EventError) SlotMismatch = Class.new(EventError) CategoryMismatch = Class.new(EventError) - UnknownAggregationOperator = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze - UNION_OF_AGGREGATED_METRICS = 'OR' - INTERSECTION_OF_AGGREGATED_METRICS = 'AND' - ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -90,37 +85,40 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) + hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end if eligible_for_totals?(events_names) - event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current) - event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current) + event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) + event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) end category_results["#{category}"] = event_results end end - def known_event?(event_name) - event_for(event_name).present? + def weekly_time_range + { start_date: 7.days.ago.to_date, end_date: Date.current } end - def aggregated_metrics_monthly_data - aggregated_metrics_data(4.weeks.ago.to_date) + def monthly_time_range + { start_date: 4.weeks.ago.to_date, end_date: Date.current } end - def aggregated_metrics_weekly_data - aggregated_metrics_data(7.days.ago.to_date) + def known_event?(event_name) + event_for(event_name).present? end def known_events @known_events ||= load_events(KNOWN_EVENTS_PATH) end - def aggregated_metrics - @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end end private @@ -139,93 +137,6 @@ module Gitlab Plan.all_plans end - def aggregated_metrics_data(start_date) - aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) - - weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) - end - end - - def calculate_count_for_aggregation(aggregation, start_date:, end_date:) - case aggregation[:operator] - when UNION_OF_AGGREGATED_METRICS - calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - when INTERSECTION_OF_AGGREGATED_METRICS - calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - else - raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" - end - end - - # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle - # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 - def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) - # calculate power of intersection of all given metrics from inclusion exclusion principle - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - - # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... - subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - - # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_events = begin - subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ - calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) - end - - # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, - # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - subset_powers_size_even = subset_powers_data.size.even? - - # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => - sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) - - # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| - sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) - end - - def sum_subset_powers(subset_powers_data, subset_powers_size_even) - sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| - (index + 1).odd? ? value : -value - end - - (subset_powers_size_even ? -1 : 1) * sum_without_sign - end - - def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - subset_sizes = (1..(event_names.size - 1)) - - subset_sizes.map do |subset_size| - if subset_size > 1 - # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) - event_names.combination(subset_size).sum do |events_subset| - subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ - calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) - end - else - # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... - event_names.sum do |event| - subset_powers_cache[subset_size][event] ||= \ - unique_events(event_names: event, start_date: start_date, end_date: end_date) - end - end - end - end - - def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) - raise AggregationMismatch, events unless events_same_aggregation?(events) - end - end - def count_unique_events(event_names:, start_date:, end_date:, context: '') events = events_for(Array(event_names).map(&:to_s)) @@ -340,12 +251,6 @@ module Gitlab end.flatten end - def validate_aggregation_operator!(operator) - return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) - - raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") - end - def weekly_redis_keys(events:, start_date:, end_date:, context: '') end_date = end_date.end_of_week - 1.week (start_date.to_date..end_date.to_date).map do |date| diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 4cbde0c0372..413b5076a20 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -268,6 +268,16 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_web_performance_widget_total +- name: i_testing_group_code_coverage_project_click_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_project_click_total +- name: i_testing_load_performance_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_load_performance_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -476,6 +486,16 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_reopen_mr +- name: i_code_review_user_resolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_resolve_thread +- name: i_code_review_user_unresolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unresolve_thread - name: i_code_review_user_merge_mr redis_slot: code_review category: code_review @@ -521,6 +541,26 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment +- name: i_code_review_user_add_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_add_suggestion +- name: i_code_review_user_apply_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_apply_suggestion +- name: i_code_review_user_assigned + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_assigned +- name: i_code_review_user_review_requested + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_review_requested # Terraform - name: p_terraform_state_api_unique_users category: terraform @@ -568,3 +608,9 @@ redis_slot: ci_templates aggregation: weekly feature_flag: usage_data_track_ci_templates_unique_projects +# Pipeline Authoring +- name: o_pipeline_authoring_unique_users_committing_ciconfigfile + category: pipeline_authoring + redis_slot: pipeline_authoring + aggregation: weekly + feature_flag: usage_data_unique_users_committing_ciconfigfile diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml new file mode 100644 index 00000000000..bf292047da0 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -0,0 +1,326 @@ +--- +- name: i_quickactions_approve + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_single + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_multiple + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_self + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_award + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_board_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clear_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_close + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_confidential + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_issue + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_create_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_done + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_draft + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_due + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_duplicate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_label + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_lock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_merge + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_promote + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_publish + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_rebase + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relabel + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_due_date + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_time_spent + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reopen + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_shrug + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_subtract + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_add + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_submit_review + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_subscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tableflip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tag + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_target_branch + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_title + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_todo + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unsubscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_wip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 11d59257ed9..1985ac0695b 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -18,6 +18,12 @@ module Gitlab MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' + MR_ADD_SUGGESTION_ACTION = 'i_code_review_user_add_suggestion' + MR_APPLY_SUGGESTION_ACTION = 'i_code_review_user_apply_suggestion' + MR_RESOLVE_THREAD_ACTION = 'i_code_review_user_resolve_thread' + MR_UNRESOLVE_THREAD_ACTION = 'i_code_review_user_unresolve_thread' + MR_ASSIGNED_USERS_ACTION = 'i_code_review_user_assigned' + MR_REVIEW_REQUESTED_USERS_ACTION = 'i_code_review_user_review_requested' class << self def track_mr_diffs_action(merge_request:) @@ -45,6 +51,14 @@ module Gitlab track_unique_action_by_user(MR_REOPEN_ACTION, user) end + def track_resolve_thread_action(user:) + track_unique_action_by_user(MR_RESOLVE_THREAD_ACTION, user) + end + + def track_unresolve_thread_action(user:) + track_unique_action_by_user(MR_UNRESOLVE_THREAD_ACTION, user) + end + def track_create_comment_action(note:) track_unique_action_by_user(MR_CREATE_COMMENT_ACTION, note.author) track_multiline_unique_action(MR_CREATE_MULTILINE_COMMENT_ACTION, note) @@ -68,6 +82,22 @@ module Gitlab track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) end + def track_add_suggestion_action(user:) + track_unique_action_by_user(MR_ADD_SUGGESTION_ACTION, user) + end + + def track_apply_suggestion_action(user:) + track_unique_action_by_user(MR_APPLY_SUGGESTION_ACTION, user) + end + + def track_users_assigned_to_mr(users:) + track_unique_action_by_users(MR_ASSIGNED_USERS_ACTION, users) + end + + def track_users_review_requested(users:) + track_unique_action_by_users(MR_REVIEW_REQUESTED_USERS_ACTION, users) + end + private def track_unique_action_by_merge_request(action, merge_request) @@ -80,6 +110,12 @@ module Gitlab track_unique_action(action, user.id) end + def track_unique_action_by_users(action, users) + return if users.blank? + + track_unique_action(action, users.map(&:id)) + end + def track_unique_action(action, value) Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb new file mode 100644 index 00000000000..f757b51f73c --- /dev/null +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module QuickActionActivityUniqueCounter + class << self + # Tracks the quick action with name `name`. + # `args` is expected to be a single string, will be split internally when necessary. + def track_unique_action(name, args:, user:) + return unless Feature.enabled?(:usage_data_track_quickactions, default_enabled: :yaml) + return unless user + + args ||= '' + name = prepare_name(name, args) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:"i_quickactions_#{name}", values: user.id) + end + + private + + def prepare_name(name, args) + case name + when 'assign' + event_name_for_assign(args) + when 'copy_metadata' + event_name_for_copy_metadata(args) + when 'remove_reviewer' + 'unassign_reviewer' + when 'request_review', 'reviewer' + 'assign_reviewer' + when 'spend' + event_name_for_spend(args) + when 'unassign' + event_name_for_unassign(args) + when 'unlabel', 'remove_label' + event_name_for_unlabel(args) + else + name + end + end + + def event_name_for_assign(args) + args = args.split + + if args.count == 1 && args.first == 'me' + 'assign_self' + elsif args.count == 1 + 'assign_single' + else + 'assign_multiple' + end + end + + def event_name_for_copy_metadata(args) + if args.start_with?('#') + 'copy_metadata_issue' + else + 'copy_metadata_merge_request' + end + end + + def event_name_for_spend(args) + if args.start_with?('-') + 'spend_subtract' + else + 'spend_add' + end + end + + def event_name_for_unassign(args) + if args.present? + 'unassign_specific' + else + 'unassign_all' + end + end + + def event_name_for_unlabel(args) + if args.present? + 'unlabel_specific' + else + 'unlabel_all' + end + end + end + end + end +end diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index e783ac785cc..5087020affe 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate)(\s+only)?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|sass))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 784a6686962..c92865636d0 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -153,7 +153,13 @@ module Gitlab def extended(mod = nil) super - queue_verification(mod.singleton_class) if mod + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + is_not_concern_hack = + (mod.is_a?(Class) || !name&.end_with?('::ClassMethods')) + + if mod && is_not_concern_hack + queue_verification(mod.singleton_class) + end end def queue_verification(base, verify: false) @@ -174,7 +180,7 @@ module Gitlab end def self.verify! - extensions.values.each(&:verify!) + extensions.each_value(&:verify!) end end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index baccadd9594..64be5c54f0f 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,6 +39,9 @@ module Gitlab FALLBACK = -1 DISTRIBUTED_HLL_FALLBACK = -2 + ALL_TIME_PERIOD_HUMAN_NAME = "all_time" + WEEKLY_PERIOD_HUMAN_NAME = "weekly" + MONTHLY_PERIOD_HUMAN_NAME = "monthly" def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -61,10 +64,13 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::PostgresHll::BatchDistinctCounter + buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter .new(relation, column) .execute(batch_size: batch_size, start: start, finish: finish) - .estimated_distinct_count + + yield buckets if block_given? + + buckets.estimated_distinct_count rescue ActiveRecord::StatementInvalid FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue @@ -74,6 +80,27 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK end + def save_aggregated_metrics(metric_name:, time_period:, recorded_at_timestamp:, data:) + unless data.is_a? ::Gitlab::Database::PostgresHll::Buckets + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(StandardError.new("Unsupported data type: #{data.class}")) + return + end + + # the longest recorded usage ping generation time for gitlab.com + # was below 40 hours, there is added error margin of 20 h + usage_ping_generation_period = 80.hours + + # add timestamp at the end of the key to avoid stale keys if + # usage ping job is retried + redis_key = "#{metric_name}_#{time_period_to_human_name(time_period)}-#{recorded_at_timestamp}" + + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_key, data.to_json, ex: usage_ping_generation_period) + end + rescue ::Redis::CommandError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + def sum(relation, column, batch_size: nil, start: nil, finish: nil) Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) rescue ActiveRecord::StatementInvalid @@ -125,6 +152,20 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end + def time_period_to_human_name(time_period) + return ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank? + + date_range = time_period.values[0] + start_date = date_range.first.to_date + end_date = date_range.last.to_date + + if (end_date - start_date).to_i > 7 + MONTHLY_PERIOD_HUMAN_NAME + else + WEEKLY_PERIOD_HUMAN_NAME + end + end + private def prometheus_client(verify:) diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb deleted file mode 100644 index b0974e02edd..00000000000 --- a/lib/gitlab_danger.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -class GitlabDanger - LOCAL_RULES ||= %w[ - changes_size - documentation - frozen_string - duplicate_yarn_dependencies - prettier - eslint - karma - database - commit_messages - product_intelligence - utility_css - pajamas - pipeline - ].freeze - - CI_ONLY_RULES ||= %w[ - metadata - changelog - specs - roulette - ce_ee_vue_templates - sidekiq_queues - specialization_labels - ci_templates - ].freeze - - MESSAGE_PREFIX = '==>'.freeze - - attr_reader :gitlab_danger_helper - - def initialize(gitlab_danger_helper) - @gitlab_danger_helper = gitlab_danger_helper - end - - def self.local_warning_message - "#{MESSAGE_PREFIX} Only the following Danger rules can be run locally: #{LOCAL_RULES.join(', ')}" - end - - def self.success_message - "#{MESSAGE_PREFIX} No Danger rule violations!" - end - - def rule_names - ci? ? LOCAL_RULES | CI_ONLY_RULES : LOCAL_RULES - end - - def html_link(str) - self.ci? ? gitlab_danger_helper.html_link(str) : str - end - - def ci? - !gitlab_danger_helper.nil? - end -end diff --git a/lib/peek/views/external_http.rb b/lib/peek/views/external_http.rb new file mode 100644 index 00000000000..bd0e4c64127 --- /dev/null +++ b/lib/peek/views/external_http.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Peek + module Views + class ExternalHttp < DetailedView + DEFAULT_THRESHOLDS = { + calls: 10, + duration: 1000, + individual_call: 100 + }.freeze + + THRESHOLDS = { + production: { + calls: 10, + duration: 1000, + individual_call: 100 + } + }.freeze + + def key + 'external-http' + end + + def results + super.merge(calls: calls) + end + + def self.thresholds + @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) + end + + def format_call_details(call) + full_path = generate_path(call) + super.merge( + label: "#{call[:method]} #{full_path}", + code: code(call), + proxy: proxy(call), + error: error(call) + ) + end + + private + + def duration + ::Gitlab::Metrics::Subscribers::ExternalHttp.duration * 1000 + end + + def calls + ::Gitlab::Metrics::Subscribers::ExternalHttp.request_count + end + + def call_details + ::Gitlab::Metrics::Subscribers::ExternalHttp.detail_store + end + + def proxy(call) + if call[:proxy_host].present? + "Proxied via #{call[:proxy_host]}:#{call[:proxy_port]}" + else + nil + end + end + + def code(call) + if call[:code].present? + "Response status: #{call[:code]}" + else + nil + end + end + + def error(call) + if call[:exception_object].present? + "Exception: #{call[:exception_object]}" + else + nil + end + end + + def generate_path(call) + uri = URI("") + uri.scheme = call[:scheme] + # The host can be a domain, IPv4 or IPv6. + # Ruby handle IPv6 for us at + # https://github.com/ruby/ruby/blob/v2_6_0/lib/uri/generic.rb#L662 + uri.hostname = call[:host] + uri.port = call[:port] + uri.path = call[:path] + uri.query = call[:query] + + uri.to_s + rescue URI::Error + 'unknown' + end + end + end +end diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb index 0dbe0cdf882..3c83ca21123 100644 --- a/lib/release_highlights/validator/entry.rb +++ b/lib/release_highlights/validator/entry.rb @@ -5,7 +5,7 @@ module ReleaseHighlights include ActiveModel::Validations include ActiveModel::Validations::Callbacks - PACKAGES = %w(Core Starter Premium Ultimate).freeze + PACKAGES = %w(Free Premium Ultimate).freeze attr_reader :entry diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index e7e0d4e471f..8f18d6433e0 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -8,9 +8,9 @@ module Rouge # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # # [+tag+] The tag (language) of the lexer used to generate the formatted tokens - def initialize(tag: nil) + def initialize(options = {}) @line_number = 1 - @tag = tag + @tag = options[:tag] end def stream(tokens) diff --git a/lib/security/ci_configuration/sast_build_actions.rb b/lib/security/ci_configuration/sast_build_actions.rb new file mode 100644 index 00000000000..b2d684bc1e1 --- /dev/null +++ b/lib/security/ci_configuration/sast_build_actions.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class SastBuildActions + SAST_DEFAULT_ANALYZERS = 'bandit, brakeman, eslint, flawfinder, gosec, kubesec, nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, sobelow, spotbugs' + + def initialize(auto_devops_enabled, params, existing_gitlab_ci_content) + @auto_devops_enabled = auto_devops_enabled + @variables = variables(params) + @existing_gitlab_ci_content = existing_gitlab_ci_content || {} + @default_sast_values = default_sast_values(params) + @default_values_overwritten = false + end + + def generate + action = @existing_gitlab_ci_content.present? ? 'update' : 'create' + + update_existing_content! + + [{ action: action, file_path: '.gitlab-ci.yml', content: prepare_existing_content, default_values_overwritten: @default_values_overwritten }] + end + + private + + def variables(params) + # This early return is necessary for supporting REST API. + # Will be removed during the implementation of + # https://gitlab.com/gitlab-org/gitlab/-/issues/246737 + return params unless params['global'].present? + + collect_values(params, 'value') + end + + def default_sast_values(params) + collect_values(params, 'defaultValue') + end + + def collect_values(config, key) + global_variables = config['global']&.to_h { |k| [k['field'], k[key]] } || {} + pipeline_variables = config['pipeline']&.to_h { |k| [k['field'], k[key]] } || {} + + analyzer_variables = collect_analyzer_values(config, key) + + global_variables.merge!(pipeline_variables).merge!(analyzer_variables) + end + + def collect_analyzer_values(config, key) + analyzer_variables = analyzer_variables_for(config, key) + analyzer_variables['SAST_EXCLUDED_ANALYZERS'] = if key == 'value' + config['analyzers'] + &.reject {|a| a['enabled'] } + &.collect {|a| a['name'] } + &.sort + &.join(', ') + else + '' + end + + analyzer_variables + end + + def analyzer_variables_for(config, key) + config['analyzers'] + &.select {|a| a['enabled'] && a['variables'] } + &.flat_map {|a| a['variables'] } + &.collect {|v| [v['field'], v[key]] }.to_h + end + + def update_existing_content! + @existing_gitlab_ci_content['stages'] = set_stages + @existing_gitlab_ci_content['variables'] = set_variables(global_variables, @existing_gitlab_ci_content) + @existing_gitlab_ci_content['sast'] = set_sast_block + @existing_gitlab_ci_content['include'] = set_includes + + @existing_gitlab_ci_content.select! { |k, v| v.present? } + @existing_gitlab_ci_content['sast'].select! { |k, v| v.present? } + end + + def set_includes + includes = @existing_gitlab_ci_content['include'] || [] + includes = includes.is_a?(Array) ? includes : [includes] + includes << { 'template' => template } + includes.uniq + end + + def set_stages + existing_stages = @existing_gitlab_ci_content['stages'] || [] + base_stages = @auto_devops_enabled ? auto_devops_stages : ['test'] + (existing_stages + base_stages + [sast_stage]).uniq + end + + def auto_devops_stages + auto_devops_template = YAML.safe_load( Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content ) + auto_devops_template['stages'] + end + + def sast_stage + @variables['stage'].presence ? @variables['stage'] : 'test' + end + + def set_variables(variables, hash_to_update = {}) + hash_to_update['variables'] ||= {} + + variables.each do |key| + if @variables[key].present? && @variables[key].to_s != @default_sast_values[key].to_s + hash_to_update['variables'][key] = @variables[key] + @default_values_overwritten = true + else + hash_to_update['variables'].delete(key) + end + end + + hash_to_update['variables'] + end + + def set_sast_block + sast_content = @existing_gitlab_ci_content['sast'] || {} + sast_content['variables'] = set_variables(sast_variables) + sast_content['stage'] = sast_stage + sast_content.select { |k, v| v.present? } + end + + def prepare_existing_content + content = @existing_gitlab_ci_content.to_yaml + content = remove_document_delimeter(content) + + content.prepend(sast_comment) + end + + def remove_document_delimeter(content) + content.gsub(/^---\n/, '') + end + + def sast_comment + <<~YAML + # You can override the included template(s) by including variable overrides + # See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings + # Note that environment variables can be set in several places + # See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables + YAML + end + + def template + return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled + + 'Security/SAST.gitlab-ci.yml' + end + + def global_variables + %w( + SECURE_ANALYZERS_PREFIX + ) + end + + def sast_variables + %w( + SAST_ANALYZER_IMAGE_TAG + SAST_EXCLUDED_PATHS + SEARCH_MAX_DEPTH + SAST_EXCLUDED_ANALYZERS + SAST_BRAKEMAN_LEVEL + SAST_BANDIT_EXCLUDED_PATHS + SAST_FLAWFINDER_LEVEL + SAST_GOSEC_LEVEL + ) + end + end + end +end diff --git a/lib/tasks/benchmark.rake b/lib/tasks/benchmark.rake new file mode 100644 index 00000000000..6deafb2c351 --- /dev/null +++ b/lib/tasks/benchmark.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :benchmark do + desc 'Benchmark | Banzai pipeline/filters' + RSpec::Core::RakeTask.new(:banzai) do |t| + t.pattern = 'spec/benchmarks/banzai_benchmark.rb' + ENV['BENCHMARK'] = '1' + end +end diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake index 6e90229830d..176693031d3 100644 --- a/lib/tasks/frontend.rake +++ b/lib/tasks/frontend.rake @@ -5,7 +5,7 @@ unless Rails.env.production? directories = %w[spec] directories << 'ee/spec' if Gitlab.ee? directory_glob = "{#{directories.join(',')}}" - args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/*.rb") + args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/**/*.rb") ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index a56a0435673..6c3a7a77e0e 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -56,7 +56,7 @@ namespace :gitlab do task orphan_job_artifact_files: :gitlab_environment do warn_user_is_not_gitlab - cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(limit: limit, dry_run: dry_run?, niceness: niceness, logger: logger) + cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(dry_run: dry_run?, niceness: niceness, logger: logger) cleaner.run! if dry_run? @@ -78,8 +78,7 @@ namespace :gitlab do cleaner = Gitlab::Cleanup::OrphanLfsFileReferences.new( project, dry_run: dry_run?, - logger: logger, - limit: limit + logger: logger ) cleaner.run! @@ -162,10 +161,6 @@ namespace :gitlab do ENV['DEBUG'].present? end - def limit - ENV['LIMIT']&.to_i - end - def niceness ENV['NICENESS'].presence end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index e15cbb4e32e..59c57a66928 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -6,30 +6,20 @@ namespace :gitlab do task migrate_legacy_storage: :gitlab_environment do logger = Logger.new(STDOUT) logger.info('Starting to migrate legacy pages storage to zip deployments') - processed_projects = 0 - ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch| - batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum| - project = metadatum.project + result = ::Pages::MigrateFromLegacyStorageService.new(logger, migration_threads, batch_size).execute - result = nil - time = Benchmark.realtime do - result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute - end - processed_projects += 1 + logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") + logger.info("- The #{result[:migrated]} projects migrated successfully") + logger.info("- The #{result[:errored]} projects failed to be migrated") + end - if result[:status] == :success - logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds") - else - logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}") - end - rescue => e - logger.error("#{e.message} project_id: #{project&.id}") - Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) - end + def migration_threads + ENV.fetch('PAGES_MIGRATION_THREADS', '3').to_i + end - logger.info("#{processed_projects} pages projects are processed") - end + def batch_size + ENV.fetch('PAGES_MIGRATION_BATCH_SIZE', '10').to_i end end end diff --git a/lib/tasks/gitlab/password.rake b/lib/tasks/gitlab/password.rake new file mode 100644 index 00000000000..02c28578a2a --- /dev/null +++ b/lib/tasks/gitlab/password.rake @@ -0,0 +1,31 @@ +# frozen_string_literal: true +namespace :gitlab do + namespace :password do + desc "GitLab | Password | Reset a user's password" + task :reset, [:username] => :environment do |_, args| + username = args[:username] || Gitlab::TaskHelpers.prompt('Enter username: ') + abort('Username can not be empty.') if username.blank? + + user = User.find_by(username: username) + abort("Unable to find user with username #{username}.") unless user + + password = Gitlab::TaskHelpers.prompt_for_password + password_confirm = Gitlab::TaskHelpers.prompt_for_password('Confirm password: ') + + user.password = password + user.password_confirmation = password_confirm + user.send_only_admin_changed_your_password_notification! + + unless user.save + message = <<~EOF + Unable to change password of the user with username #{username}. + #{user.errors.full_messages.to_sentence} + EOF + + abort(message) + end + + puts "Password successfully updated for user with username #{username}." + end + end +end diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake index ed2e88692d5..b55f82480e1 100644 --- a/lib/tasks/gitlab/snippets.rake +++ b/lib/tasks/gitlab/snippets.rake @@ -13,7 +13,7 @@ namespace :gitlab do raise "Please supply the list of ids through the SNIPPET_IDS env var" end - raise "Invalid limit value" if limit == 0 + raise "Invalid limit value" if snippet_task_limit == 0 if migration_running? raise "There are already snippet migrations running. Please wait until they are finished." @@ -41,8 +41,8 @@ namespace :gitlab do end end - if ids.size > limit - raise "The number of ids provided is higher than #{limit}. You can update this limit by using the env var `LIMIT`" + if ids.size > snippet_task_limit + raise "The number of ids provided is higher than #{snippet_task_limit}. You can update this limit by using the env var `LIMIT`" end ids @@ -68,14 +68,14 @@ namespace :gitlab do # bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50 desc 'GitLab | Show non migrated snippets' task list_non_migrated: :environment do - raise "Invalid limit value" if limit == 0 + raise "Invalid limit value" if snippet_task_limit == 0 non_migrated_count = non_migrated_snippets.count if non_migrated_count == 0 puts "All snippets have been successfully migrated" else puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n" - puts non_migrated_snippets.limit(limit).pluck(:id).join(',') + puts non_migrated_snippets.limit(snippet_task_limit).pluck(:id).join(',') end end @@ -84,7 +84,7 @@ namespace :gitlab do end # There are problems with the specs if we memoize this value - def limit + def snippet_task_limit ENV['LIMIT'] ? ENV['LIMIT'].to_i : DEFAULT_LIMIT end end diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake new file mode 100644 index 00000000000..a9c16049240 --- /dev/null +++ b/lib/tasks/gitlab/terraform/migrate.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'logger' + +desc "GitLab | Terraform | Migrate Terraform states to remote storage" +namespace :gitlab do + namespace :terraform_states do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of Terraform states to object storage') + + begin + Gitlab::Terraform::StateMigrationHelper.migrate_to_remote_storage do |state_version| + message = "Transferred Terraform state version ID #{state_version.id} (#{state_version.terraform_state.name}/#{state_version.version}) to object storage" + + logger.info(message) + end + rescue => e + logger.error("Failed to migrate: #{e.message}") + end + end + end +end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index d6f5661d5eb..0e729fa8833 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -21,5 +21,11 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(result.attributes) end + + desc 'GitLab | UsageData | Generate metrics dictionary' + task generate_metrics_dictionary: :environment do + items = Gitlab::Usage::MetricDefinition.definitions + Gitlab::Usage::Docs::Renderer.new(items).write + end end end diff --git a/lib/tasks/gitlab_danger.rake b/lib/tasks/gitlab_danger.rake index e75539f048c..deff6484231 100644 --- a/lib/tasks/gitlab_danger.rake +++ b/lib/tasks/gitlab_danger.rake @@ -1,6 +1,6 @@ desc 'Run local Danger rules' task :danger_local do - require 'gitlab_danger' + require_relative '../../tooling/gitlab_danger' require 'gitlab/popen' puts("#{GitlabDanger.local_warning_message}\n") |