diff options
Diffstat (limited to 'lib/gitlab')
271 files changed, 5628 insertions, 1918 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 111e18b2076..a36d551d1d7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -136,6 +136,7 @@ module Gitlab Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end + # rubocop: disable CodeReuse/ActiveRecord def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) @@ -146,7 +147,9 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def personal_access_token_check(password) return unless password.present? @@ -156,6 +159,7 @@ module Gitlab Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end + # rubocop: enable CodeReuse/ActiveRecord def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, [:api]) @@ -177,6 +181,7 @@ module Gitlab end.uniq end + # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? @@ -192,6 +197,7 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end + # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index eeab7791643..f323d2e0f7a 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -92,12 +92,12 @@ module Gitlab if provider Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) else Gitlab::AppLogger.info( "Account is not provided by LDAP, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) end end @@ -107,7 +107,7 @@ module Gitlab Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + "unblocking GitLab user \"#{user.name}\" (#{user.email})" ) end end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 922d0567d99..3c21ddf3241 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -11,11 +11,13 @@ module Gitlab extend ::Gitlab::Utils::Override class << self + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider(uid, provider) identity = ::Identity.with_extern_uid(provider, uid).take identity && identity.user end + # rubocop: enable CodeReuse/ActiveRecord end def save diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 589e8062226..2b4f6ed75e5 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -112,11 +112,13 @@ module Gitlab build_new_user end + # rubocop: disable CodeReuse/ActiveRecord def find_by_email return unless auth_hash.has_attribute?(:email) ::User.find_by(email: auth_hash.email.downcase) end + # rubocop: enable CodeReuse/ActiveRecord def auto_link_ldap_user? Gitlab.config.omniauth.auto_link_ldap_user @@ -180,10 +182,12 @@ module Gitlab @auth_hash = AuthHash.new(auth_hash) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity&.user end + # rubocop: enable CodeReuse/ActiveRecord def build_new_user user_params = user_attributes.merge(skip_confirmation: true) diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb index f79ce6bb809..8ae29a02a13 100644 --- a/lib/gitlab/auth/omniauth_identity_linker_base.rb +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -33,11 +33,13 @@ module Gitlab @changed = identity.save end + # rubocop: disable CodeReuse/ActiveRecord def identity @identity ||= current_user.identities .with_extern_uid(provider, uid) .first_or_initialize(extern_uid: uid) end + # rubocop: enable CodeReuse/ActiveRecord def provider oauth['provider'] diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index c7993665421..064cba43278 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -71,6 +71,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || @@ -81,6 +82,7 @@ module Gitlab # Expiration, revocation and scopes are verified in `validate_access_token!` PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) end + # rubocop: enable CodeReuse/ActiveRecord def find_oauth_access_token token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb new file mode 100644 index 00000000000..0d333e47e7b --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # For this background migration to work, the table that is migrated _has_ to + # have an `id` column as the primary key. Additionally, the encrypted column + # should be managed by attr_encrypted, and map to an attribute with the same + # name as the unencrypted column (i.e., the unencrypted column should be + # shadowed). + # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + class EncryptColumns + def perform(model, attributes, from, to) + model = model.constantize if model.is_a?(String) + attributes = expand_attributes(model, Array(attributes).map(&:to_sym)) + + model.transaction do + # Use SELECT ... FOR UPDATE to prevent the value being changed while + # we are encrypting it + relation = model.where(id: from..to).lock + + relation.each do |instance| + encrypt!(instance, attributes) + end + end + end + + private + + # Build a hash of { attribute => encrypted column name } + def expand_attributes(klass, attributes) + expanded = attributes.flat_map do |attribute| + attr_config = klass.encrypted_attributes[attribute] + crypt_column_name = attr_config&.fetch(:attribute) + + raise "Couldn't determine encrypted column for #{klass}##{attribute}" if + crypt_column_name.nil? + + [attribute, crypt_column_name] + end + + Hash[*expanded] + end + + # Generate ciphertext for each column and update the database + def encrypt!(instance, attributes) + to_clear = attributes + .map { |plain, crypt| apply_attribute!(instance, plain, crypt) } + .compact + .flat_map { |plain| [plain, nil] } + + to_clear = Hash[*to_clear] + + if instance.changed? + instance.save! + instance.update_columns(to_clear) + end + end + + def apply_attribute!(instance, plain_column, crypt_column) + plaintext = instance[plain_column] + ciphertext = instance[crypt_column] + + # No need to do anything if the plaintext is nil, or an encrypted + # value already exists + return nil unless plaintext.present? && !ciphertext.present? + + # attr_encrypted will calculate and set the expected value for us + instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend + + plain_column + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb new file mode 100644 index 00000000000..5cd638083b0 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/ClassLength + +module Gitlab + module BackgroundMigration + ## + # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts` + class MigrateLegacyArtifacts + FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive'] + METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata'] + LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path'] + + def perform(start_id, stop_id) + ActiveRecord::Base.transaction do + insert_archives(start_id, stop_id) + insert_metadatas(start_id, stop_id) + delete_legacy_artifacts(start_id, stop_id) + end + end + + private + + def insert_archives(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_file, + artifacts_size, + COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), + #{ARCHIVE_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE}) + SQL + end + + def insert_metadatas(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_metadata, + NULL, + COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), + #{METADATA_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND artifacts_metadata <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE}) + SQL + end + + def delete_legacy_artifacts(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + ci_builds + SET + artifacts_file = NULL, + artifacts_file_store = NULL, + artifacts_size = NULL, + artifacts_metadata = NULL, + artifacts_metadata_store = NULL + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb new file mode 100644 index 00000000000..bb76eb8ed48 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` and `url` columns + class WebHook < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'web_hooks' + self.inheritance_column = :_type_disabled + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + + attr_encrypted :url, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb new file mode 100644 index 00000000000..036fe641757 --- /dev/null +++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateExternalPipelineSource + module Migratable + class Pipeline < ActiveRecord::Base + self.table_name = 'ci_pipelines' + + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6 + } + end + end + + class CommitStatus < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } + scope :of_type, -> (type) { where('type=?', type) } + end + end + + def perform(start_id, stop_id) + external_pipelines(start_id, stop_id) + .update_all(source: Migratable::Pipeline.sources[:external]) + end + + private + + def external_pipelines(start_id, stop_id) + Migratable::Pipeline.where(id: (start_id..stop_id)) + .where( + 'EXISTS (?) AND NOT EXISTS (?)', + Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), + Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 68f3fa62170..9941c2fe6d9 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation +# rubocop:disable Metrics/ClassLength module Gitlab module BackgroundMigration @@ -49,11 +50,14 @@ module Gitlab private def remove_non_members_todos(project_id) - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - .each_batch(of: 5000) do |batch| - batch.delete_all - end + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id) + else + unauthorized_project_todos(project_id) + .each_batch(of: 5000) do |batch| + batch.delete_all + end + end end def remove_confidential_issue_todos(project_id) @@ -86,10 +90,13 @@ module Gitlab next if target_types.empty? - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - .where(target_type: target_types) - .delete_all + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id, target_types) + else + unauthorized_project_todos(project_id) + .where(target_type: target_types) + .delete_all + end end end @@ -100,6 +107,65 @@ module Gitlab def authorized_users(project_id) ProjectAuthorization.select(:user_id).where(project_id: project_id) end + + def unauthorized_project_todos(project_id) + Todo.where(project_id: project_id) + .where('user_id NOT IN (?)', authorized_users(project_id)) + end + + def batch_remove_todos_cte(project_id, target_types = nil) + loop do + count = remove_todos_cte(project_id, target_types) + + break if count == 0 + end + end + + def remove_todos_cte(project_id, target_types = nil) + sql = [] + sql << with_all_todos_sql(project_id, target_types) + sql << as_deleted_sql + sql << "SELECT count(*) FROM deleted" + + result = Todo.connection.exec_query(sql.join(' ')) + result.rows[0][0].to_i + end + + def with_all_todos_sql(project_id, target_types = nil) + if target_types + table = Arel::Table.new(:todos) + in_target = table[:target_type].in(target_types) + target_types_sql = " AND #{in_target.to_sql}" + end + + <<-SQL + WITH all_todos AS ( + SELECT id + FROM "todos" + WHERE "todos"."project_id" = #{project_id} + AND (user_id NOT IN ( + SELECT "project_authorizations"."user_id" + FROM "project_authorizations" + WHERE "project_authorizations"."project_id" = #{project_id}) + #{target_types_sql} + ) + ), + SQL + end + + def as_deleted_sql + <<-SQL + deleted AS ( + DELETE FROM todos + WHERE id IN ( + SELECT id + FROM all_todos + LIMIT 5000 + ) + RETURNING id + ) + SQL + end end end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 778d78185ff..16fd6f01495 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -36,6 +36,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def raw_coverage return unless @pipeline @@ -47,6 +48,7 @@ module Gitlab .try(:coverage) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index 5fee7a93475..d1d9b7949f5 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -18,11 +18,13 @@ module Gitlab 'pipeline' end + # rubocop: disable CodeReuse/ActiveRecord def status @project.pipelines .where(sha: @sha) .latest_status(@ref) || 'unknown' end + # rubocop: enable CodeReuse/ActiveRecord def metadata @metadata ||= Pipeline::Metadata.new(self) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index fa0186c854c..a7dfccea2f6 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -43,6 +43,7 @@ module Gitlab find_user_id(username) || project.creator_id end + # rubocop: disable CodeReuse/ActiveRecord def find_user_id(username) return nil unless username @@ -53,6 +54,7 @@ module Gitlab .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord def repo @repo ||= client.repo(project.import_source) @@ -68,6 +70,7 @@ module Gitlab errors << { type: :wiki, errors: e.message } end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues_enabled? @@ -101,6 +104,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_comments(issue, gitlab_issue) client.issue_comments(repo, issue.iid).each do |comment| diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index b591d94668f..15aa4739ee9 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :recover_missing_commits attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + attr_accessor :logger REMOTE_NAME = 'bitbucket_server'.freeze BATCH_SIZE = 100 @@ -36,6 +37,7 @@ module Gitlab @errors = [] @users = {} @temp_branches = [] + @logger = Gitlab::Import::Logger.build end def execute @@ -44,6 +46,8 @@ module Gitlab delete_temp_branches handle_errors + log_info(stage: "complete") + true end @@ -118,15 +122,21 @@ module Gitlab client.create_branch(project_key, repository_slug, branch_name, sha) branches_created << temp_branch rescue BitbucketServer::Connection::ConnectionError => e - Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}") + log_warn(message: "Unable to recreate branch", sha: sha, error: e.message) end end end def import_repository + log_info(stage: 'import_repository', message: 'starting import') + project.ensure_repository project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) + + log_info(stage: 'import_repository', message: 'finished import') rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e + log_error(stage: 'import_repository', message: 'failed import', error: e.message) + # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true # 2. Retried import, repo is broken or not imported but +exists?+ still returns true @@ -157,7 +167,10 @@ module Gitlab begin import_bitbucket_pull_request(pull_request) rescue StandardError => e - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } + backtrace = Gitlab::Profiler.clean_backtrace(e.backtrace) + log_error(stage: 'import_pull_requests', iid: pull_request.iid, error: e.message, backtrace: backtrace) + + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end end end @@ -169,12 +182,15 @@ module Gitlab client.delete_branch(project_key, repository_slug, branch.name, branch.sha) project.repository.delete_branch(branch.name) rescue BitbucketServer::Connection::ConnectionError => e + log_error(stage: 'delete_temp_branches', branch: branch.name, error: e.message) @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } end end end def import_bitbucket_pull_request(pull_request) + log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) + description = '' description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) description += pull_request.description if pull_request.description @@ -201,9 +217,13 @@ module Gitlab merge_request = creator.execute(attributes) import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + + log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) end def import_pull_request_comments(pull_request, merge_request) + log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) + comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) merge_event = other_activities.find(&:merge_event?) @@ -213,9 +233,17 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + + log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, + merge_event_found: merge_event.present?, + inline_comments_count: inline_comments.count, + standalone_pr_comments: pr_comments.count) end + # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) + log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + committer = merge_event.committer_email user_id = gitlab_user_id(committer) @@ -223,9 +251,14 @@ module Gitlab merge_request.update({ merge_commit_sha: merge_event.merge_commit }) metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) metric.update(merged_by_id: user_id, merged_at: timestamp) + + log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) end + # rubocop: enable CodeReuse/ActiveRecord def import_inline_comments(inline_comments, merge_request) + log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + inline_comments.each do |comment| position = build_position(merge_request, comment) parent = create_diff_note(merge_request, comment, position) @@ -238,6 +271,8 @@ module Gitlab create_diff_note(merge_request, reply, position, discussion_id) end end + + log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) end def create_diff_note(merge_request, comment, position, discussion_id = nil) @@ -252,11 +287,14 @@ module Gitlab return note end + log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + # Bitbucket Server supports the ability to comment on any line, not just the # line in the diff. If we can't add the note as a DiffNote, fallback to creating # a regular note. create_fallback_diff_note(merge_request, comment, position) rescue StandardError => e + log_error(stage: 'create_diff_note', comment_id: comment.id, error: e.message) errors << { type: :pull_request, id: comment.id, errors: e.message } nil end @@ -294,7 +332,8 @@ module Gitlab merge_request.notes.create!(pull_request_comment_attributes(replies)) end rescue StandardError => e - errors << { type: :pull_request, iid: comment.id, errors: e.message } + log_error(stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message) + errors << { type: :pull_request, comment_id: comment.id, errors: e.message } end end end @@ -324,6 +363,26 @@ module Gitlab updated_at: comment.updated_at } end + + def log_info(details) + logger.info(log_base_data.merge(details)) + end + + def log_error(details) + logger.error(log_base_data.merge(details)) + end + + def log_warn(details) + logger.warn(log_base_data.merge(details)) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end end end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index 671b8e7e1b1..b96e161a5b6 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -26,8 +26,8 @@ module Gitlab define_method(method_name) do |*args| store = - if RequestStore.active? - RequestStore.store + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore.store else ivar_name = # ! and ? cannot be used as ivar name "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index 22310e313ac..7e0c34aada3 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -43,6 +43,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def lfs_file_locks_validation lambda do |paths| lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first @@ -52,6 +53,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def path_validations validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index b816a8f00cd..3f7adecc621 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -6,6 +6,7 @@ module Gitlab @newrev = newrev end + # rubocop: disable CodeReuse/ActiveRecord def objects_missing? return false unless @newrev && @project.lfs_enabled? @@ -20,6 +21,7 @@ module Gitlab existing_count != new_lfs_pointers.count end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 849848515da..86f4aaeb4d3 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -7,12 +7,14 @@ module Gitlab @project = project end + # rubocop: disable CodeReuse/ActiveRecord def match? @project.merge_requests .with_state(:locked) .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) .exists? end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 428c0505808..85072a072d6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -96,12 +96,14 @@ module Gitlab blank_node? || @entries.include?(@path.to_s) end + # rubocop: disable CodeReuse/ActiveRecord def total_size descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end end + # rubocop: enable CodeReuse/ActiveRecord def path @path.to_s diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb new file mode 100644 index 00000000000..7bf51519752 --- /dev/null +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Policy + class Changes < Policy::Specification + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true unless pipeline.branch_updated? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 46ed330dbbf..7b7354bce16 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -2,12 +2,14 @@ module Gitlab module Ci module Charts module DailyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) query .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.day @@ -15,6 +17,7 @@ module Gitlab end module MonthlyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) if Gitlab::Database.postgresql? query @@ -27,6 +30,7 @@ module Gitlab .count(:created_at) end end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.month @@ -46,6 +50,7 @@ module Gitlab collect end + # rubocop: disable CodeReuse/ActiveRecord def collect query = project.pipelines .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection @@ -64,6 +69,7 @@ module Gitlab current += interval_step end end + # rubocop: enable CodeReuse/ActiveRecord end class YearChart < Chart diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 66ac4a40616..fe98d25af29 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,15 +1,22 @@ module Gitlab module Ci - ## + # # Base GitLab CI Configuration facade # class Config - # EE would override this and utilize opts argument + ConfigError = Class.new(StandardError) + def initialize(config, opts = {}) - @config = Loader.new(config).load! + @config = Config::Extendable + .new(build_config(config, opts)) + .to_hash @global = Entry::Global.new(@config) @global.compose! + rescue Loader::FormatError, Extendable::ExtensionError => e + raise Config::ConfigError, e.message + rescue ::Gitlab::Ci::External::Processor::FileError => e + raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message end def valid? @@ -58,6 +65,24 @@ module Gitlab def jobs @global.jobs_value end + + private + + def build_config(config, opts = {}) + initial_config = Loader.new(config).load! + project = opts.fetch(:project, nil) + + if project + process_external_files(initial_config, project, opts) + else + initial_config + end + end + + def process_external_files(config, project, opts) + sha = opts.fetch(:sha) { project.repository.root_ref_sha } + ::Gitlab::Ci::External::Processor.new(config, project, sha).perform + end end end end diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 7cddd2c7b7e..697f622c45e 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -24,6 +24,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) return unless valid? @@ -41,6 +42,7 @@ module Gitlab entry.compose!(deps) end end + # rubocop: enable CodeReuse/ActiveRecord class_methods do def nodes @@ -49,12 +51,14 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def entry(key, entry, metadata) factory = Entry::Factory.new(entry) .with(description: metadata[:description]) (@nodes ||= {}).merge!(key.to_sym => factory) end + # rubocop: enable CodeReuse/ActiveRecord def helpers(*nodes) nodes.each do |symbol| diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index a4ec8f0ff2f..04077fa7a61 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -45,6 +45,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) @@ -53,6 +54,7 @@ module Gitlab @entries[:jobs] = factory.create! end + # rubocop: enable CodeReuse/ActiveRecord def compose_deprecated_entries! ## diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 91aac6df4b1..f290ff3a565 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -9,9 +9,10 @@ module Gitlab include Configurable include Attributable - ALLOWED_KEYS = %i[tags script only except type image services allow_failure - type stage when artifacts cache dependencies before_script - after_script variables environment coverage retry].freeze + ALLOWED_KEYS = %i[tags script only except type image services + allow_failure type stage when start_in artifacts cache + dependencies before_script after_script variables + environment coverage retry extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -27,12 +28,16 @@ module Gitlab greater_than_or_equal_to: 0, less_than_or_equal_to: 2 } validates :when, - inclusion: { in: %w[on_success on_failure always manual], + inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ - 'always or manual' } + 'always, manual or delayed' } validates :dependencies, array_of_strings: true + validates :extends, type: String end + + validates :start_in, duration: { limit: '1 day' }, if: :delayed? + validates :start_in, absence: true, unless: :delayed? end entry :before_script, Entry::Script, @@ -81,7 +86,8 @@ module Gitlab :cache, :image, :services, :only, :except, :variables, :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies, :retry + attributes :script, :tags, :allow_failure, :when, :dependencies, + :retry, :extends, :start_in def compose!(deps = nil) super do @@ -111,6 +117,10 @@ module Gitlab self.when == 'manual' end + def delayed? + self.when == 'delayed' + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 5671a09480b..96b6f2e5d6c 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -26,6 +26,7 @@ module Gitlab name.to_s.start_with?('.') end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| @@ -45,6 +46,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index a78a85397bd..a3d4432be82 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -11,6 +11,15 @@ module Gitlab false end + def validate_duration_limit(value, limit) + return false unless value.is_a?(String) + + ChronicDuration.parse(value).second.from_now < + ChronicDuration.parse(limit).second.from_now + rescue ChronicDuration::DurationParseError + false + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 09e8e52b60f..c92562f8c85 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,17 +25,19 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes, :variables + ALLOWED_KEYS = %i[refs kubernetes variables changes].freeze + attributes :refs, :kubernetes, :variables, :changes validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes variables] + validates :config, allowed_keys: ALLOWED_KEYS validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true + validates :changes, array_of_strings: true end def variables_expressions_syntax diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 5963f3eb90c..98f12c226b3 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -9,7 +11,7 @@ module Gitlab include Validatable include Attributable - ALLOWED_KEYS = %i[junit].freeze + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast].freeze attributes ALLOWED_KEYS @@ -19,6 +21,11 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :codequality, array_of_strings_or_string: true + validates :sast, array_of_strings_or_string: true + validates :dependency_scanning, array_of_strings_or_string: true + validates :container_scanning, array_of_strings_or_string: true + validates :dast, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b3c889ee92f..f6b4ba7843e 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -49,6 +49,12 @@ module Gitlab unless validate_duration(value) record.errors.add(attribute, 'should be a duration') end + + if options[:limit] + unless validate_duration_limit(value, options[:limit]) + record.errors.add(attribute, 'should not exceed the limit') + end + end end end diff --git a/lib/gitlab/ci/config/extendable.rb b/lib/gitlab/ci/config/extendable.rb new file mode 100644 index 00000000000..a43901c69fe --- /dev/null +++ b/lib/gitlab/ci/config/extendable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + include Enumerable + + ExtensionError = Class.new(StandardError) + + def initialize(hash) + @hash = hash.to_h.deep_dup + + each { |entry| entry.extend! if entry.extensible? } + end + + def each + @hash.each_key do |key| + yield Extendable::Entry.new(key, @hash) + end + end + + def to_hash + @hash.to_h + end + end + end + end +end diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb new file mode 100644 index 00000000000..7793db09d33 --- /dev/null +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + class Entry + InvalidExtensionError = Class.new(Extendable::ExtensionError) + CircularDependencyError = Class.new(Extendable::ExtensionError) + NestingTooDeepError = Class.new(Extendable::ExtensionError) + + MAX_NESTING_LEVELS = 10 + + attr_reader :key + + def initialize(key, context, parent = nil) + @key = key + @context = context + @parent = parent + + unless @context.key?(@key) + raise StandardError, 'Invalid entry key!' + end + end + + def extensible? + value.is_a?(Hash) && value.key?(:extends) + end + + def value + @value ||= @context.fetch(@key) + end + + def base_hash! + @base ||= Extendable::Entry + .new(extends_key, @context, self) + .extend! + end + + def extends_key + value.fetch(:extends).to_s.to_sym if extensible? + end + + def ancestors + @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key) + end + + def extend! + return value unless extensible? + + if unknown_extension? + raise Entry::InvalidExtensionError, + "#{key}: unknown key in `extends`" + end + + if invalid_base? + raise Entry::InvalidExtensionError, + "#{key}: invalid base hash in `extends`" + end + + if nesting_too_deep? + raise Entry::NestingTooDeepError, + "#{key}: nesting too deep in `extends`" + end + + if circular_dependency? + raise Entry::CircularDependencyError, + "#{key}: circular dependency detected in `extends`" + end + + @context[key] = base_hash!.deep_merge(value) + end + + private + + def nesting_too_deep? + ancestors.count > MAX_NESTING_LEVELS + end + + def circular_dependency? + ancestors.include?(key) + end + + def unknown_extension? + !@context.key?(extends_key) + end + + def invalid_base? + !@context[extends_key].is_a?(Hash) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb new file mode 100644 index 00000000000..f4da07b0b02 --- /dev/null +++ b/lib/gitlab/ci/external/file/base.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Base + YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze + + def initialize(location, opts = {}) + @location = location + end + + def valid? + location.match(YAML_WHITELIST_EXTENSION) && content + end + + def content + raise NotImplementedError, 'content must be implemented and return a string or nil' + end + + def error_message + raise NotImplementedError, 'error_message must be implemented and return a string' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb new file mode 100644 index 00000000000..1aa7f687507 --- /dev/null +++ b/lib/gitlab/ci/external/file/local.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Local < Base + attr_reader :location, :project, :sha + + def initialize(location, opts = {}) + super + + @project = opts.fetch(:project) + @sha = opts.fetch(:sha) + end + + def content + @content ||= fetch_local_content + end + + def error_message + "Local file '#{location}' is not valid." + end + + private + + def fetch_local_content + project.repository.blob_data_at(sha, location) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb new file mode 100644 index 00000000000..59bb3e8999e --- /dev/null +++ b/lib/gitlab/ci/external/file/remote.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Remote < Base + include Gitlab::Utils::StrongMemoize + attr_reader :location + + def content + return @content if defined?(@content) + + @content = strong_memoize(:content) do + begin + Gitlab::HTTP.get(location) + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError + nil + end + end + end + + def error_message + "Remote file '#{location}' is not valid." + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb new file mode 100644 index 00000000000..58bd6a19acf --- /dev/null +++ b/lib/gitlab/ci/external/mapper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Mapper + def initialize(values, project, sha) + @locations = Array(values.fetch(:include, [])) + @project = project + @sha = sha + end + + def process + locations.map { |location| build_external_file(location) } + end + + private + + attr_reader :locations, :project, :sha + + def build_external_file(location) + if ::Gitlab::UrlSanitizer.valid?(location) + Gitlab::Ci::External::File::Remote.new(location) + else + options = { project: project, sha: sha } + Gitlab::Ci::External::File::Local.new(location, options) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb new file mode 100644 index 00000000000..76cf3ce89f9 --- /dev/null +++ b/lib/gitlab/ci/external/processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Processor + FileError = Class.new(StandardError) + + def initialize(values, project, sha) + @values = values + @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process + @content = {} + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + @content.deep_merge!(content_of(external_file)) + end + + append_inline_content + remove_include_keyword + end + + private + + attr_reader :values, :external_files, :content + + def validate_external_file(external_file) + unless external_file.valid? + raise FileError, external_file.error_message + end + end + + def content_of(external_file) + Gitlab::Ci::Config::Loader.new(external_file.content).load! + end + + def append_inline_content + @content.deep_merge!(@values) + end + + def remove_include_keyword + content.delete(:include) + content + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb deleted file mode 100644 index a4eccc08dfc..00000000000 --- a/lib/gitlab/ci/parsers.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Ci - module Parsers - def self.fabricate!(file_type) - "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new - end - end - end -end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb deleted file mode 100644 index 3c4668ec13b..00000000000 --- a/lib/gitlab/ci/parsers/junit.rb +++ /dev/null @@ -1,69 +0,0 @@ -module Gitlab - module Ci - module Parsers - class Junit - attr_reader :data - - JunitParserError = Class.new(StandardError) - - def parse!(xml_data, test_suite) - @data = Hash.from_xml(xml_data) - - each_suite do |testcases| - testcases.each do |testcase| - test_case = create_test_case(testcase) - test_suite.add_test_case(test_case) - end - end - rescue REXML::ParseException => e - raise JunitParserError, "XML parsing failed: #{e.message}" - rescue => e - raise JunitParserError, "JUnit parsing failed: #{e.message}" - end - - private - - def each_suite - testsuites.each do |testsuite| - yield testcases(testsuite) - end - end - - def testsuites - if data['testsuites'] - data['testsuites']['testsuite'] - else - [data['testsuite']] - end - end - - def testcases(testsuite) - if testsuite['testcase'].is_a?(Array) - testsuite['testcase'] - else - [testsuite['testcase']] - end - end - - def create_test_case(data) - if data['failure'] - status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED - system_output = data['failure'] - else - status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS - system_output = nil - end - - ::Gitlab::Ci::Reports::TestCase.new( - classname: data['classname'], - name: data['name'], - file: data['file'], - execution_time: data['time'], - status: status, - system_output: system_output - ) - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/test.rb b/lib/gitlab/ci/parsers/test.rb new file mode 100644 index 00000000000..c6bc9662b07 --- /dev/null +++ b/lib/gitlab/ci/parsers/test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + ParserNotFoundError = Class.new(StandardError) + + PARSERS = { + junit: ::Gitlab::Ci::Parsers::Test::Junit + }.freeze + + def self.fabricate!(file_type) + PARSERS.fetch(file_type.to_sym).new + rescue KeyError + raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb new file mode 100644 index 00000000000..5d7d9a751d8 --- /dev/null +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + class Junit + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + root = Hash.from_xml(xml_data) + + all_cases(root) do |test_case| + test_case = create_test_case(test_case) + test_suite.add_test_case(test_case) + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def all_cases(root, parent = nil, &blk) + return unless root.present? + + [root].flatten.compact.map do |node| + next unless node.is_a?(Hash) + + # we allow only one top-level 'testsuites' + all_cases(node['testsuites'], root, &blk) unless parent + + # we require at least one level of testsuites or testsuite + each_case(node['testcase'], &blk) if parent + + # we allow multiple nested 'testsuite' (eg. PHPUnit) + all_cases(node['testsuite'], root, &blk) + end + end + + def each_case(testcase, &blk) + return unless testcase.present? + + [testcase].flatten.compact.map(&blk) + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index f4c8d5342c1..02493c7fe02 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -5,6 +5,7 @@ module Gitlab class Create < Chain::Base include Chain::Helpers + # rubocop: disable CodeReuse/ActiveRecord def perform! ::Ci::Pipeline.transaction do pipeline.save! @@ -23,6 +24,7 @@ module Gitlab rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end + # rubocop: enable CodeReuse/ActiveRecord def break? !pipeline.persisted? diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index 469fc094cc8..30701e1de1b 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -86,6 +86,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) status = %w[success failed running canceled] builds = pipeline.builds.latest @@ -93,6 +94,7 @@ module Gitlab from_builds(builds) end + # rubocop: enable CodeReuse/ActiveRecord def from_builds(builds) now = Time.now @@ -134,9 +136,11 @@ module Gitlab Period.new(previous.first, [previous.last, current.last].max) end + # rubocop: disable CodeReuse/ActiveRecord def process_duration(periods) periods.sum(&:duration) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index c6e732e68eb..c87bdb4a8a2 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -12,13 +12,17 @@ module Gitlab test_suites[suite_name] ||= TestSuite.new(suite_name) end + # rubocop: disable CodeReuse/ActiveRecord def total_time test_suites.values.sum(&:total_time) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def total_count test_suites.values.sum(&:total_count) end + # rubocop: enable CodeReuse/ActiveRecord def total_status if failed_count > 0 || error_count > 0 @@ -30,7 +34,9 @@ module Gitlab TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}_count") do + # rubocop: disable CodeReuse/ActiveRecord test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index c0943f5a51a..726c6a11a81 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -29,7 +29,9 @@ module Gitlab %w(total_count resolved_count failed_count).each do |method| define_method(method) do + # rubocop: disable CodeReuse/ActiveRecord suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index b722d0ba735..b5f15397c0f 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -21,9 +21,11 @@ module Gitlab @total_time += test_case.execution_time end + # rubocop: disable CodeReuse/ActiveRecord def total_count test_cases.values.sum(&:count) end + # rubocop: enable CodeReuse/ActiveRecord def total_status if failed_count > 0 || error_count > 0 diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 2b26ebb45a1..4a74d6d6ed1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -5,6 +5,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, + Status::Build::Scheduled, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, @@ -14,6 +15,7 @@ module Gitlab Status::Build::Retryable], [Status::Build::Failed], [Status::Build::FailedAllowed, + Status::Build::Unschedule, Status::Build::Play, Status::Build::Stop], [Status::Build::Action], diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 508b4814631..50b0d044265 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -10,9 +10,12 @@ module Gitlab stuck_or_timeout_failure: 'stuck or timeout failure', runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', - runner_unsupported: 'unsupported runner' + runner_unsupported: 'unsupported runner', + stale_schedule: 'stale schedule' }.freeze + private_constant :REASONS + def status_tooltip base_message end @@ -25,6 +28,10 @@ module Gitlab build.failed? end + def self.reasons + REASONS + end + private def base_message @@ -36,7 +43,7 @@ module Gitlab end def failure_reason_message - REASONS.fetch(subject.failure_reason.to_sym) + self.class.reasons.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb new file mode 100644 index 00000000000..eebb3f761c5 --- /dev/null +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + module Status + module Build + class Scheduled < Status::Extended + def illustration + { + image: 'illustrations/illustrations_scheduled-job_countdown.svg', + size: 'svg-394', + title: _("This is a scheduled to run in ") + " #{execute_in}", + content: _("This job will automatically run after it's timer finishes. " \ + "Often they are used for incremental roll-out deploys " \ + "to production environments. When unscheduled it converts " \ + "into a manual action.") + } + end + + def status_tooltip + "scheduled manual action (#{execute_in})" + end + + def self.matches?(build, user) + build.scheduled? && build.scheduled_at + end + + private + + include TimeHelper + + def execute_in + remaining_seconds = [0, subject.scheduled_at - Time.now].max + duration_in_numbers(remaining_seconds, true) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb new file mode 100644 index 00000000000..e1b7b83428c --- /dev/null +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + module Status + module Build + class Unschedule < Status::Extended + def label + 'unschedule action' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'time-out' + end + + def action_title + 'Unschedule' + end + + def action_button_title + _('Unschedule job') + end + + def action_path + unschedule_project_job_path(subject.project, subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 17f9a75f436..00d8f01cbdc 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -5,6 +5,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, + Status::Pipeline::Scheduled, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/pipeline/scheduled.rb b/lib/gitlab/ci/status/pipeline/scheduled.rb new file mode 100644 index 00000000000..9ec6994bd2f --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/scheduled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Scheduled < Status::Extended + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|waiting for delayed job') + end + + def self.matches?(pipeline, user) + pipeline.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb new file mode 100644 index 00000000000..542100e41da --- /dev/null +++ b/lib/gitlab/ci/status/scheduled.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + class Scheduled < Status::Core + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|scheduled') + end + + def icon + 'status_scheduled' + end + + def favicon + 'favicon_status_scheduled' + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml new file mode 100644 index 00000000000..5f9d54ff574 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Read more about this script on this blog post https://about.gitlab.com/2016/11/30/setting-up-gitlab-ci-for-android-projects/, by Greyson Parrelli +image: openjdk:8-jdk + +variables: + ANDROID_COMPILE_SDK: "25" + ANDROID_BUILD_TOOLS: "24.0.0" + ANDROID_SDK_TOOLS: "24.4.1" + +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - wget --quiet --output-document=android-sdk.tgz https://dl.google.com/android/android-sdk_r${ANDROID_SDK_TOOLS}-linux.tgz + - tar --extract --gzip --file=android-sdk.tgz + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter android-${ANDROID_COMPILE_SDK} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter platform-tools + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter build-tools-${ANDROID_BUILD_TOOLS} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-android-m2repository + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-google_play_services + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-m2repository + - export ANDROID_HOME=$PWD/android-sdk-linux + - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ + - chmod +x ./gradlew + +stages: + - build + - test + +build: + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +unitTests: + stage: test + script: + - ./gradlew test + +functionalTests: + stage: test + script: + - wget --quiet --output-document=android-wait-for-emulator https://raw.githubusercontent.com/travis-ci/travis-cookbooks/0f497eb71291b52a703143c5cd63a217c8766dc9/community-cookbooks/android-sdk/files/default/android-wait-for-emulator + - chmod +x android-wait-for-emulator + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter sys-img-x86-google_apis-${ANDROID_COMPILE_SDK} + - echo no | android-sdk-linux/tools/android create avd -n test -t android-${ANDROID_COMPILE_SDK} --abi google_apis/x86 + - android-sdk-linux/tools/emulator64-x86 -avd test -no-window -no-audio & + - ./android-wait-for-emulator + - adb shell input keyevent 82 + - ./gradlew cAT diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..72547c1b407 --- /dev/null +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,897 @@ +# Auto DevOps +# This CI/CD configuration provides a standard pipeline for +# * building a Docker image (using a buildpack if necessary), +# * storing the image in the container registry, +# * running tests from a buildpack, +# * running code quality analysis, +# * creating a review app for each topic branch, +# * and continuous deployment to production +# +# Test jobs may be disabled by setting environment variables: +# * test: TEST_DISABLED +# * code_quality: CODE_QUALITY_DISABLED +# * license_management: LICENSE_MANAGEMENT_DISABLED +# * performance: PERFORMANCE_DISABLED +# * sast: SAST_DISABLED +# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED +# * container_scanning: CONTAINER_SCANNING_DISABLED +# * dast: DAST_DISABLED +# * review: REVIEW_DISABLED +# * stop_review: REVIEW_DISABLED +# +# In order to deploy, you must have a Kubernetes cluster configured either +# via a project integration, or via group/project variables. +# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project +# level, or manually added below. +# +# Continuous deployment to production is enabled by default. +# If you want to deploy to staging first, set STAGING_ENABLED environment variable. +# If you want to enable incremental rollout, either manual or time based, +# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed". +# If you want to use canary deployments, set CANARY_ENABLED environment variable. +# +# If Auto DevOps fails to detect the proper buildpack, or if you want to +# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the +# repository URL of the buildpack. +# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 +# If you need multiple buildpacks, add a file to your project called +# `.buildpacks` that contains the URLs, one on each line, in order. +# Note: Auto CI does not work with multiple buildpacks yet + +image: alpine:latest + +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # AUTO_DEVOPS_DOMAIN: domain.example.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + + KUBERNETES_VERSION: 1.8.6 + HELM_VERSION: 2.11.0 + + DOCKER_DRIVER: overlay2 + +stages: + - build + - test + - review + - dast + - staging + - canary + - production + - incremental rollout 10% + - incremental rollout 25% + - incremental rollout 50% + - incremental rollout 100% + - performance + - cleanup + +build: + stage: build + image: docker:stable-git + services: + - docker:stable-dind + script: + - setup_docker + - build + only: + - branches + +test: + services: + - postgres:latest + variables: + POSTGRES_DB: test + stage: test + image: gliderlabs/herokuish:latest + script: + - setup_test_db + - cp -R . /tmp/app + - /bin/herokuish buildpack test + only: + - branches + except: + variables: + - $TEST_DISABLED + +code_quality: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - code_quality + artifacts: + paths: [gl-code-quality-report.json] + only: + - branches + except: + variables: + - $CODE_QUALITY_DISABLED + +license_management: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - license_management + artifacts: + paths: [gl-license-management-report.json] + only: + - branches + only: + variables: + - $GITLAB_FEATURES =~ /\blicense_management\b/ + except: + variables: + - $LICENSE_MANAGEMENT_DISABLED + +performance: + stage: performance + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - performance + artifacts: + paths: + - performance.json + - sitespeed-results/ + only: + refs: + - branches + kubernetes: active + except: + variables: + - $PERFORMANCE_DISABLED + +sast: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - sast + artifacts: + paths: [gl-sast-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ + except: + variables: + - $SAST_DISABLED + +dependency_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - dependency_scanning + artifacts: + paths: [gl-dependency-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED + +container_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - container_scanning + artifacts: + paths: [gl-container-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast_container\b/ + except: + variables: + - $CONTAINER_SCANNING_DISABLED + +dast: + stage: dast + allow_failure: true + image: registry.gitlab.com/gitlab-org/security-products/zaproxy + variables: + POSTGRES_DB: "false" + script: + - dast + artifacts: + paths: [gl-dast-report.json] + only: + refs: + - branches + kubernetes: active + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED + +review: + stage: review + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +stop_review: + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - install_dependencies + - initialize_tiller + - delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual + allow_failure: true + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + stage: staging + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + +# Canaries are also disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + stage: canary + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $CANARY_ENABLED + +.production: &production_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - delete canary + - delete rollout + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +production: + <<: *production_template + only: + refs: + - master + kubernetes: active + except: + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +production_manual: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy rollout $ROLLOUT_PERCENTAGE + - scale stable $((100-ROLLOUT_PERCENTAGE)) + - delete canary + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + when: manual + # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "manual" + - $INCREMENTAL_ROLLOUT_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + when: delayed + start_in: 5 minutes + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false + +# --------------------------------------------------------------------------- + +.auto_devops: &auto_devops | + # Auto DevOps variables and functions + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + function registry_login() { + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi + } + + function container_scanning() { + registry_login + + docker run -d --name db arminc/clair-db:latest + docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1 + apk add -U wget ca-certificates + docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} + wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + mv clair-scanner_linux_amd64 clair-scanner + chmod +x clair-scanner + touch clair-whitelist.yml + retries=0 + echo "Waiting for clair daemon to start" + while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done + ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + } + + function code_quality() { + docker run --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code + } + + function license_management() { + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" + LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + docker run --volume "$PWD:/code" \ + "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code + } + + function sast() { + case "$CI_SERVER_VERSION" in + *-ee) + + # Deprecation notice for CONFIDENCE_LEVEL variable + if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then + SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" + echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" + fi + + docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function dependency_scanning() { + case "$CI_SERVER_VERSION" in + *-ee) + docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name="$CI_ENVIRONMENT_SLUG" + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + name="$name-$track" + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + } + + function scale() { + track="${1-stable}" + percentage="${2-100}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm upgrade --reuse-values \ + --wait \ + --set replicaCount="$replicas" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + } + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add glibc-2.28-r0.apk + rm glibc-2.28-r0.apk + + curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ + helm version --client + tiller -version + + curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function setup_docker() { + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + } + + function setup_test_db() { + if [ -z ${KUBERNETES_PORT+x} ]; then + DB_HOST=postgres + else + DB_HOST=localhost + fi + export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + auto_chart_name=${auto_chart_name%.tar.gz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab https://charts.gitlab.io + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + function check_kube_domain() { + if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi + } + + function build() { + registry_login + + if [[ -f Dockerfile ]]; then + echo "Building Dockerfile-based application..." + docker build \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg http_proxy="$http_proxy" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg FTP_PROXY="$FTP_PROXY" \ + --build-arg ftp_proxy="$ftp_proxy" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg no_proxy="$no_proxy" \ + -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" . + else + echo "Building Heroku-based application using gliderlabs/herokuish docker image..." + docker run -i \ + -e BUILDPACK_URL \ + -e HTTP_PROXY \ + -e http_proxy \ + -e HTTPS_PROXY \ + -e https_proxy \ + -e FTP_PROXY \ + -e ftp_proxy \ + -e NO_PROXY \ + -e no_proxy \ + --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + + echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..." + docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + fi + + echo "Pushing to GitLab Container Registry..." + docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + echo "" + } + + function initialize_tiller() { + echo "Checking Tiller..." + + export HELM_HOST=":44134" + tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & + echo "Tiller is listening on ${HELM_HOST}" + + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ + --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function dast() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir /zap/wrk/ + /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true + cp /zap/wrk/gl-dast-report.json . + } + + function performance() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir gitlab-exporter + wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + + mkdir sitespeed-results + + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + else + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + fi + + mv sitespeed-results/data/performance.json performance.json + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + + function delete() { + track="${1-stable}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm delete --purge "$name" + fi + } + +before_script: + - *auto_devops diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml new file mode 100644 index 00000000000..2d218b2e164 --- /dev/null +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -0,0 +1,35 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# you can delete this line if you're not using Docker +image: busybox:latest + +before_script: + - echo "Before script section" + - echo "For example you might run an update here or install a build dependency" + - echo "Or perhaps you might print out some debugging details" + +after_script: + - echo "After script section" + - echo "For example you might do some cleanup here" + +build1: + stage: build + script: + - echo "Do your build here" + +test1: + stage: test + script: + - echo "Do a test here" + - echo "For example run a test suite" + +test2: + stage: test + script: + - echo "Do another parallel test here" + - echo "For example run a lint test" + +deploy1: + stage: deploy + script: + - echo "Do your deploy here" diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml new file mode 100644 index 00000000000..c83c49d8c95 --- /dev/null +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -0,0 +1,26 @@ +# use the official gcc image, based on debian +# can use verions as well, like gcc:5.2 +# see https://hub.docker.com/_/gcc/ +image: gcc + +build: + stage: build + # instead of calling g++ directly you can also use some build toolkit like make + # install the necessary build tools when needed + # before_script: + # - apt update && apt -y install make autoconf + script: + - g++ helloworld.cpp -o mybinary + artifacts: + paths: + - mybinary + # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time + # cache: + # paths: + # - "*.o" + +# run tests using the binary built before +test: + stage: test + script: + - ./runmytests.sh diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml new file mode 100644 index 00000000000..4d5b6484d6e --- /dev/null +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -0,0 +1,51 @@ +# This file uses Test Kitchen with the kitchen-dokken driver to +# perform functional testing. Doing so requires that your runner be a +# Docker runner configured for privileged mode. Please see +# https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode +# for help configuring your runner properly, or, if you want to switch +# to a different driver, see http://kitchen.ci/docs/drivers + +image: "chef/chefdk" +services: + - docker:dind + +variables: + DOCKER_HOST: "tcp://docker:2375" + KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml" + +stages: + - lint + - unit + - functional + +foodcritic: + stage: lint + script: + - chef exec foodcritic . + +cookstyle: + stage: lint + script: + - chef exec cookstyle . + +chefspec: + stage: unit + script: + - chef exec rspec spec + +# Set up your test matrix here. Example: +#verify-centos-6: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-6 --destroy=always +# +#verify-centos-7: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-7 --destroy=always diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml new file mode 100644 index 00000000000..f066285b1ad --- /dev/null +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Based on openjdk:8, already includes lein +image: clojure:lein-2.7.0 +# If you need to configure a database, add a `services` section here +# See https://docs.gitlab.com/ce/ci/services/postgres.html +# Make sure you configure the connection as well + +before_script: + # If you need to install any external applications, like a + # postgres client, you may want to uncomment the line below: + # + #- apt-get update -y + # + # Retrieve project dependencies + # Do this on before_script since it'll be shared between both test and + # any production sections a user adds + - lein deps + +test: + script: + # If you need to run any migrations or configure the database, this + # would be the point to do it. + - lein test diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml new file mode 100644 index 00000000000..36386a19fdc --- /dev/null +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/crystallang/crystal/ +image: "crystallang/crystal:latest" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# variables: +# POSTGRES_DB: database_name + +# Cache shards in between builds +cache: + paths: + - lib + +# This is a basic example for a shard or script which doesn't use +# services such as redis or postgres +before_script: + - apt-get update -qq && apt-get install -y -qq libxml2-dev + - crystal -v # Print out Crystal version for debugging + - shards + +# If you are using built-in Crystal Spec. +spec: + script: + - crystal spec + +# If you are using minitest.cr +minitest: + script: + - crystal test/spec_test.cr # change to the file(s) you execute for tests diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml new file mode 100644 index 00000000000..57afcbbe8b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -0,0 +1,49 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python +image: python:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - ~/.cache/pip/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - python -V # Print out python version for debugging + # Uncomment next line if your Django app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - pip install -r requirements.txt + +# To get Django tests to work you may need to create a settings file using +# the following DATABASES: +# +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql_psycopg2', +# 'NAME': 'ci', +# 'USER': 'postgres', +# 'PASSWORD': 'postgres', +# 'HOST': 'postgres', +# 'PORT': '5432', +# }, +# } +# +# and then adding `--settings app.settings.ci` (or similar) to the test command + +test: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - python manage.py test diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml new file mode 100644 index 00000000000..eeefadaa019 --- /dev/null +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Official docker image. +image: docker:latest + +services: + - docker:dind + +before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +build-master: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE" . + - docker push "$CI_REGISTRY_IMAGE" + only: + - master + +build: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" + except: + - master diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml new file mode 100644 index 00000000000..cf9c731637c --- /dev/null +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -0,0 +1,18 @@ +image: elixir:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +before_script: + - mix local.rebar --force + - mix local.hex --force + - mix deps.get + +mix: + script: + - mix test diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml new file mode 100644 index 00000000000..d572d7a1edc --- /dev/null +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -0,0 +1,35 @@ +image: golang:latest + +variables: + # Please edit to your GitLab project + REPO_NAME: gitlab.com/namespace/project + +# The problem is that to be able to use go get, one needs to put +# the repository in the $GOPATH. So for example if your gitlab domain +# is gitlab.com, and that your repository is namespace/project, and +# the default GOPATH being /go, then you'd need to have your +# repository in /go/src/gitlab.com/namespace/project +# Thus, making a symbolic link corrects this. +before_script: + - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) + - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME + - cd $GOPATH/src/$REPO_NAME + +stages: + - test + - build + +format: + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) + +compile: + stage: build + script: + - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + artifacts: + paths: + - mybinary diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml new file mode 100644 index 00000000000..48d98dddfad --- /dev/null +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -0,0 +1,36 @@ +# This is the Gradle build system for JVM applications +# https://gradle.org/ +# https://github.com/gradle/gradle +image: gradle:alpine + +# Disable the Gradle daemon for Continuous Integration servers as correctness +# is usually a priority over speed in CI environments. Using a fresh +# runtime for each build is more reliable since the runtime is completely +# isolated from any previous builds. +variables: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + +before_script: + - export GRADLE_USER_HOME=`pwd`/.gradle + +build: + stage: build + script: gradle --build-cache assemble + cache: + key: "$CI_COMMIT_REF_NAME" + policy: push + paths: + - build + - .gradle + + +test: + stage: test + script: gradle check + cache: + key: "$CI_COMMIT_REF_NAME" + policy: pull + paths: + - build + - .gradle + diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml new file mode 100644 index 00000000000..7fc698d50cf --- /dev/null +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -0,0 +1,40 @@ +# This template uses the java:8 docker image because there isn't any +# official Grails image at this moment +# +# Grails Framework https://grails.org/ is a powerful Groovy-based web application framework for the JVM +# +# This yml works with Grails 3.x only +# Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) +# Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) +# If you use Angular profile, this yml it's prepared to work with it + +image: java:8 + +variables: + GRAILS_VERSION: "3.1.9" + GRADLE_VERSION: "2.13" + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install gradle $GRADLE_VERSION < /dev/null + - sdk use gradle $GRADLE_VERSION +# As it's not a good idea to version gradle.properties feel free to add your +# environments variable here + - echo grailsVersion=$GRAILS_VERSION > gradle.properties + - echo gradleWrapperVersion=2.14 >> gradle.properties +# refresh dependencies from your project + - ./gradlew --refresh-dependencies +# Be aware that if you are using Angular profile, +# Bower cannot be run as root if you don't allow it before. +# Feel free to remove next line if you are not using Bower + - echo {\"allow_root\":true} > /root/.bowerrc + +# This build job does the full grails pipeline +# (compile, test, integrationTest, war, assemble). +build: + script: + - ./gradlew build
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml new file mode 100644 index 00000000000..04c21b4725d --- /dev/null +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -0,0 +1,76 @@ +# This is an example .gitlab-ci.yml file to test (and optionally report the coverage +# results of) your [Julia][1] packages. Please refer to the [documentation][2] +# for more information about package development in Julia. +# +# Here, it is assumed that your Julia package is named `MyPackage`. Change it to +# whatever name you have given to your package. +# +# [1]: http://julialang.org/ +# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html + +# Below is the template to run your tests in Julia +.test_template: &test_definition + # Uncomment below if you would like to run the tests on specific references + # only, such as the branches `master`, `development`, etc. + # only: + # - master + # - development + script: + # Let's run the tests. Substitute `coverage = false` below, if you do not + # want coverage results. + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' + # Comment out below if you do not want coverage results. + - julia -e 'using Pkg; Pkg.add("Coverage"); + import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); + using Coverage; cl, tl = get_summary(process_folder()); + println("(", cl/tl*100, "%) covered")' + +# Name a test and select an appropriate image. +# images comes from Docker hub +test:0.7: + image: julia:0.7 + <<: *test_definition + +test:1.0: + image: julia:1.0 + <<: *test_definition + +# Maybe you would like to test your package against the development branch: +# test:1.1-dev (not sure there is such an image in docker, so not tested yet): +# image: julia:v1.1-dev +# # ... allowing for failures, since we are testing against the development +# # branch: +# allow_failure: true +# <<: *test_definition + +# REMARK: Do not forget to enable the coverage feature for your project, if you +# are using code coverage reporting above. This can be done by +# +# - Navigating to the `CI/CD Pipelines` settings of your project, +# - Copying and pasting the default `Simplecov` regex example provided, i.e., +# `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield. + +# Example documentation deployment +pages: + image: julia:0.7 + stage: deploy + script: + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages + artifacts: + paths: + - public + only: + - master + + +# WARNING: This template is using the `julia` images from [Docker +# Hub][3]. One can use custom Julia images and/or the official ones found +# in the same place. However, care must be taken to correctly locate the binary +# file (`/opt/julia/bin/julia` above), which is usually given on the image's +# description page. +# +# [3]: https://hub.docker.com/_/julia/ diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml new file mode 100644 index 00000000000..a4aed36889e --- /dev/null +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -0,0 +1,11 @@ +# use docker image with latex preinstalled +# since there is no official latex image, use https://github.com/blang/latex-docker +# possible alternative: https://github.com/natlownes/docker-latex +image: blang/latex + +build: + script: + - latexmk -pdf + artifacts: + paths: + - "*.pdf" diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml new file mode 100644 index 00000000000..d0cad285572 --- /dev/null +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -0,0 +1,85 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/php +image: php:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + +variables: + MYSQL_DATABASE: project_name + MYSQL_ROOT_PASSWORD: secret + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - vendor/ + - node_modules/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + # Update packages + - apt-get update -yqq + + # Prep for Node + - apt-get install gnupg -yqq + + # Upgrade to Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + + # Install dependencies + - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq + + # Install php extensions + - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache + + # Install & enable Xdebug for code coverage reports + - pecl install xdebug + - docker-php-ext-enable xdebug + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + # comment this out if you don't have a node dependency + - npm install + + # Copy over testing configuration. + # Don't forget to set the database config in .env.testing correctly + # DB_HOST=mysql + # DB_DATABASE=project_name + # DB_USERNAME=root + # DB_PASSWORD=secret + - cp .env.testing .env + + # Run npm build + # comment this out if you don't have a frontend build + # you can change this to to your frontend building script like + # npm run build + - npm run dev + + # Generate an application key. Re-cache. + - php artisan key:generate + - php artisan config:cache + + # Run database migrations. + - php artisan migrate + + # Run database seed + - php artisan db:seed + +test: + script: + # run laravel tests + - php vendor/bin/phpunit --coverage-text --colors=never + + # run frontend tests + # if you have any task for testing frontend + # set it in your package.json script + # comment this out if you don't have a frontend test + - npm test diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml new file mode 100644 index 00000000000..d61ff239e13 --- /dev/null +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -0,0 +1,102 @@ +--- +# Build JAVA applications using Apache Maven (http://maven.apache.org) +# For docker image tags see https://hub.docker.com/_/maven/ +# +# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +# +# This template will build and test your projects as well as create the documentation. +# +# * Caches downloaded dependencies and plugins between invocation. +# * Verify but don't deploy merge requests. +# * Deploy built artifacts from master branch only. +# * Shows how to use multiple jobs in test stage for verifying functionality +# with multiple JDKs. +# * Uses site:stage to collect the documentation for multi-module projects. +# * Publishes the documentation for `master` branch. + +variables: + # This will supress any download for dependencies and plugins or upload messages which would clutter the console log. + # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. + MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + # when running from the command line. + # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + +# Cache downloaded dependencies and plugins between builds. +# To keep cache across branches add 'key: "$CI_JOB_NAME"' +cache: + paths: + - .m2/repository + +# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. +# Because some enforcer rules might check dependency convergence and class duplications +# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. +.validate: &validate + stage: build + script: + - 'mvn $MAVEN_CLI_OPTS test-compile' + +# For merge requests do not `deploy` but only run `verify`. +# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +.verify: &verify + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + except: + - master + +# Validate merge requests using JDK7 +validate:jdk7: + <<: *validate + image: maven:3.3.9-jdk-7 + +# Validate merge requests using JDK8 +validate:jdk8: + <<: *validate + image: maven:3.3.9-jdk-8 + +# Verify merge requests using JDK7 +verify:jdk7: + <<: *verify + image: maven:3.3.9-jdk-7 + +# Verify merge requests using JDK8 +verify:jdk8: + <<: *verify + image: maven:3.3.9-jdk-8 + + +# For `master` branch run `mvn deploy` automatically. +# Here you need to decide whether you want to use JDK7 or 8. +# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. +# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. +# See https://maven.apache.org/settings.html +deploy:jdk8: + # Use stage test here, so the pages job may later pickup the created site. + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' + only: + - master + # Archive up the built documentation site. + artifacts: + paths: + - target/staging + image: maven:3.3.9-jdk-8 + + +pages: + image: busybox:latest + stage: deploy + script: + # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, + # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. + - mv target/staging public + dependencies: + - deploy:jdk8 + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml new file mode 100644 index 00000000000..3585f99760f --- /dev/null +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -0,0 +1,42 @@ +# This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) +# using the official mono docker image to build a visual studio project. +# +# MyProject.sln +# MyProject\ +# MyProject\ +# MyProject.csproj (console application) +# MyProject.Test\ +# MyProject.Test.csproj (test library using nuget packages "NUnit" and "NUnit.ConsoleRunner") +# +# Please find the full example project here: +# https://gitlab.com/tobiaskoch/gitlab-ci-example-mono + +# see https://hub.docker.com/_/mono/ +image: mono:latest + +stages: + - test + - deploy + +before_script: + - nuget restore -NonInteractive + +release: + stage: deploy + only: + - master + artifacts: + paths: + - build/release/MyProject.exe + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Release" /p:Platform="Any CPU" + /p:OutputPath="./../../build/release/" "MyProject.sln" + +debug: + stage: test + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" + /p:OutputPath="./../../build/debug/" "MyProject.sln" + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml new file mode 100644 index 00000000000..41de1458582 --- /dev/null +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/node/tags/ +image: node:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +test_async: + script: + - npm install + - node ./specs/start.js ./specs/async.spec.js + +test_db: + script: + - npm install + - node ./specs/start.js ./specs/db-postgres.spec.js diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml new file mode 100644 index 00000000000..290b9997084 --- /dev/null +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -0,0 +1,92 @@ +image: ayufan/openshift-cli + +stages: + - test + - review + - staging + - production + - cleanup + +variables: + OPENSHIFT_SERVER: openshift.default.svc.cluster.local + # OPENSHIFT_DOMAIN: apps.example.com + # Configure this variable in Secure Variables: + # OPENSHIFT_TOKEN: my.openshift.token + +test1: + stage: test + before_script: [] + script: + - echo run tests + +test2: + stage: test + before_script: [] + script: + - echo run tests + +.deploy: &deploy + before_script: + - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" --insecure-skip-tls-verify + - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID" + script: + - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker" + - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow" + - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST" + +review: + <<: *deploy + stage: review + variables: + APP: review-$CI_COMMIT_REF_NAME + APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + on_stop: stop-review + only: + - branches + except: + - master + +stop-review: + <<: *deploy + stage: cleanup + script: + - oc delete all -l "app=$APP" + when: manual + variables: + APP: review-$CI_COMMIT_REF_NAME + GIT_STRATEGY: none + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + only: + - branches + except: + - master + +staging: + <<: *deploy + stage: staging + variables: + APP: staging + APP_HOST: $CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + environment: + name: staging + url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + only: + - master + +production: + <<: *deploy + stage: production + variables: + APP: production + APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + when: manual + environment: + name: production + url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + only: + - master diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml new file mode 100644 index 00000000000..33f44ee9222 --- /dev/null +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Select image from https://hub.docker.com/_/php/ +image: php:7.1.1 + +# Select what we should cache between builds +cache: + paths: + - vendor/ + +before_script: +- apt-get update -yqq +- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev +# Install PHP extensions +- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache +# Install & enable Xdebug for code coverage reports +- pecl install xdebug +- docker-php-ext-enable xdebug +# Install and run Composer +- curl -sS https://getcomposer.org/installer | php +- php composer.phar install + +# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +services: + - mysql:5.7 + +# Set any variables we need +variables: + # Configure mysql environment variables (https://hub.docker.com/r/_/mysql/) + MYSQL_DATABASE: mysql_database + MYSQL_ROOT_PASSWORD: mysql_strong_password + +# Run our tests +# If Xdebug was installed you can generate a coverage report and see code coverage metrics. +test: + script: + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml new file mode 100644 index 00000000000..fa296057c72 --- /dev/null +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: + name: hashicorp/packer:1.0.4 + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +before_script: + - packer --version + +stages: + - validate + - deploy + +validate: + stage: validate + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer validate + +build: + stage: deploy + environment: production + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml new file mode 100644 index 00000000000..7fcc0b436b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/brunch +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g brunch + - brunch build --production + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml new file mode 100644 index 00000000000..791afdd23f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -0,0 +1,13 @@ +# Full project: https://gitlab.com/pages/doxygen +image: alpine + +pages: + script: + - apk update && apk add doxygen + - doxygen doxygen/Doxyfile + - mv doxygen/documentation/html/ public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml new file mode 100644 index 00000000000..9df2a4797b2 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -0,0 +1,17 @@ +image: node:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +pages: + script: + - yarn install + - ./node_modules/.bin/gatsby build --prefix-paths + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml new file mode 100644 index 00000000000..249a168aa33 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/plain-html +pages: + stage: deploy + script: + - mkdir .public + - cp -r * .public + - mv .public public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml new file mode 100644 index 00000000000..dd3ef149668 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/harp +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules + + script: + - npm install -g harp + - harp compile ./ public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml new file mode 100644 index 00000000000..02d02250bbf --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/hexo +image: node:6.10.0 + +pages: + script: + - npm install + - ./node_modules/hexo/bin/hexo generate + artifacts: + paths: + - public + cache: + paths: + - node_modules + key: project + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml new file mode 100644 index 00000000000..b8cfb0f56f6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/hugo +image: dettmering/hugo-build + +pages: + script: + - hugo + artifacts: + paths: + - public + only: + - master + +test: + script: + - hugo + except: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml new file mode 100644 index 00000000000..f5b40f2b9f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -0,0 +1,25 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +cache: + paths: + - vendor/ + +test: + stage: test + script: + - pip install hyde + - hyde gen + except: + - master + +pages: + stage: deploy + script: + - pip install hyde + - hyde gen -d public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml new file mode 100644 index 00000000000..7abfaf53e8e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -0,0 +1,32 @@ +# This template uses the java:8 docker image because there isn't any +# official JBake image at this moment +# +# JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers +# +# This yml works with jBake 2.5.1 +# Feel free to change JBAKE_VERSION version +# +# HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ + +image: java:8 + +variables: + JBAKE_VERSION: 2.5.1 + + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION + +# This build job produced the output directory of your site +pages: + script: + - jbake . public + artifacts: + paths: + - public
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml new file mode 100644 index 00000000000..37f50554036 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Template project: https://gitlab.com/pages/jekyll +# Docs: https://docs.gitlab.com/ce/pages/ +image: ruby:2.3 + +variables: + JEKYLL_ENV: production + +before_script: +- bundle install + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + except: + - master + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + only: + - master + diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml new file mode 100644 index 00000000000..0e5fb410a4e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -0,0 +1,42 @@ +# Jigsaw is a simple static sites generator with Laravel's Blade. +# +# Full project: https://github.com/tightenco/jigsaw + +image: php:7.2 + +# These folders are cached between builds +cache: + paths: + - vendor/ + - node_modules/ + +before_script: + # Update packages + - apt-get update -yqq + + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + + # Install php extensions + - docker-php-ext-install zip + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + - npm install + +pages: + script: + - npm run production + - mv build_production public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml new file mode 100644 index 00000000000..c5c44a5d86c --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +pages: + script: + - pip install lektor + - lektor build --output-path public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml new file mode 100644 index 00000000000..50e8b7ccd46 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/metalsmith +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g metalsmith + - npm install + - make build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml new file mode 100644 index 00000000000..9f4cc0574d6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Full project: https://gitlab.com/pages/middleman +image: ruby:2.3 + +cache: + paths: + - vendor + +test: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + except: + - master + +pages: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml new file mode 100644 index 00000000000..b469b316ba5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/nanoc +image: ruby:2.3 + +pages: + script: + - bundle install -j4 + - nanoc + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml new file mode 100644 index 00000000000..4762ec9acfd --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -0,0 +1,15 @@ +# Full project: https://gitlab.com/pages/octopress +image: ruby:2.3 + +pages: + script: + - apt-get update -qq && apt-get install -qq nodejs + - bundle install -j4 + - bundle exec rake generate + - mv public .public + - mv .public/octopress public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml new file mode 100644 index 00000000000..c5f3154f587 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -0,0 +1,10 @@ +# Full project: https://gitlab.com/pages/pelican +image: python:2.7-alpine + +pages: + script: + - pip install -r requirements.txt + - pelican -s publishconf.py + artifacts: + paths: + - public/ diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml new file mode 100644 index 00000000000..2e0589de652 --- /dev/null +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python/tags/ +image: python:latest + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +cache: + paths: + - .cache/pip + - venv/ + +before_script: + - python -V # Print out python version for debugging + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate + +test: + script: + - python setup.py test + - pip install tox flake8 # you can also use tox + - tox -e py36,flake8 + +run: + script: + - python setup.py bdist_wheel + # an alternative approach is to install and run: + - pip install dist/* + # run the command here + artifacts: + paths: + - dist/*.whl + +pages: + script: + - pip install sphinx sphinx-rtd-theme + - cd doc ; make html + - mv build/html/ ../public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml new file mode 100644 index 00000000000..93cb31f48c0 --- /dev/null +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -0,0 +1,54 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/ruby/tags/ +image: "ruby:2.5" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# Cache gems in between builds +cache: + paths: + - vendor/ruby + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - ruby -v # Print out ruby version for debugging + # Uncomment next line if your rails app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - gem install bundler --no-ri --no-rdoc # Bundler is not installed with the image + - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + +# Optional - Delete if not using `rubocop` +rubocop: + script: + - rubocop + +rspec: + script: + - rspec spec + +rails: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - rails db:migrate + - rails db:seed + - rails test + +# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk +# are supported too: https://github.com/travis-ci/dpl +deploy: + type: deploy + environment: production + script: + - gem install dpl + - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml new file mode 100644 index 00000000000..cab087c48c7 --- /dev/null +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -0,0 +1,23 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/rust/tags/ +image: "rust:latest" + +# Optional: Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +#services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# Optional: Install a C compiler, cmake and git into the container. +# You will often need this when you (or any of your dependencies) depends on C code. +#before_script: +#- apt-get update -yqq +#- apt-get install -yqq --no-install-recommends build-essential + +# Use cargo to test the project +test:cargo: + script: + - rustc --version && cargo --version # Print version info for debugging + - cargo test --all --verbose diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml new file mode 100644 index 00000000000..b4208ed9d7d --- /dev/null +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Official Java image. Look for the different tagged releases at +# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# but an image with a JVM speeds up the build a bit. +image: java:8 + +before_script: + # Enable the usage of sources over https + - apt-get update -yqq + - apt-get install apt-transport-https -yqq + # Add keyserver for SBT + - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + # Install SBT + - apt-get update -yqq + - apt-get install sbt -yqq + # Log the sbt version + - sbt sbt-version + +test: + script: + # Execute your project's tests + - sbt clean test diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml new file mode 100644 index 00000000000..ba8a802ba4f --- /dev/null +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ +# This file assumes an own GitLab CI runner, setup on a macOS system. +stages: + - build + - archive + +build_project: + stage: build + script: + - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty + - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 + +archive_project: + stage: archive + script: + - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName + - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" + only: + - master + artifacts: + paths: + - build/ProjectName.ipa + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml new file mode 100644 index 00000000000..7160fce26a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -0,0 +1,55 @@ +# Official image for Hashicorp's Terraform. It uses light image which is Alpine +# based as it is much lighter. +# +# Entrypoint is also needed as image by default set `terraform` binary as an +# entrypoint. +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + PLAN: plan.tfplan + +cache: + paths: + - .terraform + +before_script: + - terraform --version + - terraform init + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + artifacts: + name: plan + paths: + - $PLAN + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml new file mode 100644 index 00000000000..fc3d4ecdbba --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -0,0 +1,86 @@ +# The following script will work for any project that can be built from command line by msbuild +# It uses powershell shell executor, so you need to add the following line to your config.toml file +# (located in gitlab-runner.exe directory): +# shell = "powershell" +# +# The script is composed of 3 stages: build, test and deploy. +# +# The build stage restores NuGet packages and uses msbuild to build the exe and msi +# One major issue you'll find is that you can't build msi projects from command line +# if you use vdproj. There are workarounds building msi via devenv, but they rarely work +# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# by msbuild. +# +# The test stage runs nunit from command line against Test project inside your solution +# It also saves the resulting TestResult.xml file +# +# The deploy stage copies the exe and msi from build stage to a network drive +# You need to have the network drive mapped as Local System user for gitlab-runner service to see it +# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes + + +# place project specific paths in variables to make the rest of the script more generic +variables: + EXE_RELEASE_FOLDER: 'YourApp\bin\Release' + MSI_RELEASE_FOLDER: 'Setup\bin\Release' + TEST_FOLDER: 'Tests\bin\Release' + DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' + + NUGET_PATH: 'C:\NuGet\nuget.exe' + MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' + NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' + +stages: + - build + - test + - deploy + +build_job: + stage: build + only: + - tags # the build process will only be started by git tag commits + script: + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + +test_job: + stage: test + only: + - tags + script: + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + dependencies: + - build_job + +deploy_job: + stage: deploy + only: + - tags + script: + # Compose a folder for each release based on commit tag. + # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit' + # the artifact files will be copied to: + # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ + - '$commitSubject = git log -1 --pretty=%s' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + + # xcopy takes care of recursively creating required folders + - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' + - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' + - 'xcopy /y ".\TestResult.xml" "$deployFolder"' + + dependencies: + - build_job + - test_job +
\ No newline at end of file diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index bfe0c2a2c26..2147f62a84a 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -133,6 +133,7 @@ module Gitlab invalidate_chunk_cache end + # rubocop: disable CodeReuse/ActiveRecord def truncate(offset) raise ArgumentError, 'Outside of file' if offset > size || offset < 0 return if offset == size # Skip the following process as it doesn't affect anything @@ -148,6 +149,7 @@ module Gitlab ensure invalidate_chunk_cache end + # rubocop: enable CodeReuse/ActiveRecord def flush # no-op @@ -206,9 +208,11 @@ module Gitlab @chunks_cache = [] end + # rubocop: disable CodeReuse/ActiveRecord def current_chunk @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) end + # rubocop: enable CodeReuse/ActiveRecord def build_chunk @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) @@ -218,13 +222,17 @@ module Gitlab current_chunk || build_chunk end + # rubocop: disable CodeReuse/ActiveRecord def trace_chunks ::Ci::BuildTraceChunk.where(build: build) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def calculate_size trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index e829f2a95f8..a427aa30683 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -16,7 +16,7 @@ module Gitlab end initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e + rescue Gitlab::Ci::Config::ConfigError => e raise ValidationError, e.message end @@ -49,7 +49,8 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], - retry: job[:retry] + retry: job[:retry], + start_in: job[:start_in] }.compact } end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index f55ab535efe..82a405362c2 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -38,6 +38,7 @@ module Gitlab end # Accepts a path in the form of "#{hex_secret}/#{filename}" + # rubocop: disable CodeReuse/ActiveRecord def find_correct_path(upload_path) upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) return unless upload && upload.local? && upload.model @@ -52,6 +53,7 @@ module Gitlab # I.e. the project record might be missing, which raises an exception. nil end + # rubocop: enable CodeReuse/ActiveRecord def move_to_lost_and_found(path, dry_run) new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND) @@ -107,18 +109,22 @@ module Gitlab new(path_matched[1], path_matched[2]) end + # rubocop: disable CodeReuse/ActiveRecord def orphan? return true if full_path.nil? || upload_path.nil? # It's possible to reduce to one query, but `where_full_path_in` is complex !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader') end + # rubocop: enable CodeReuse/ActiveRecord private + # rubocop: disable CodeReuse/ActiveRecord def project_id @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 45a5aea4fcd..eba1faacc3a 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -33,6 +33,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def each_orphan_file # we want to skip files already moved to lost_and_found directory lost_dir_match = "^#{lost_and_found_dir}\/" @@ -50,6 +51,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def move_to_lost_and_found(file) new_path = "#{lost_and_found_dir}/#{file.key}" diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 4c28489f45a..1ffc2639237 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -7,9 +7,14 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor @current_user = current_user - @projects = ContributedProjectsFinder.new(contributor).execute(current_user) + @projects = if @contributor.include_private_contributions? + ContributedProjectsFinder.new(@contributor).execute(@contributor) + else + ContributedProjectsFinder.new(contributor).execute(current_user) + end end + # rubocop: disable CodeReuse/ActiveRecord def activity_dates return @activity_dates if @activity_dates.present? @@ -25,25 +30,25 @@ module Gitlab note_events = event_counts(date_from, :merge_requests) .having(action: [Event::COMMENTED]) - union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) - events = Event.find_by_sql(union.to_sql).map(&:attributes) + events = Event + .from_union([repo_events, issue_events, mr_events, note_events]) + .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def events_by_date(date) return Event.none unless can_read_cross_project? - events = Event.contributions.where(author_id: contributor.id) + Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) - - # Use visible_to_user? instead of the complicated logic in activity_dates - # because we're only viewing the events for a single day. - events.select { |event| event.visible_to_user?(current_user) } end + # rubocop: enable CodeReuse/ActiveRecord def starting_year 1.year.ago.year @@ -59,6 +64,7 @@ module Gitlab Ability.allowed?(current_user, :read_cross_project) end + # rubocop: disable CodeReuse/ActiveRecord def event_counts(date_from, feature) t = Event.arel_table @@ -87,5 +93,6 @@ module Gitlab .where(conditions) .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9147ef401da..de7c959e706 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -2,17 +2,17 @@ module Gitlab module CurrentSettings class << self def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! - end + Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end def fake_application_settings(attributes = {}) Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end + def clear_in_memory_application_settings! + @in_memory_application_settings = nil + end + def method_missing(name, *args, &block) current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index eb246d393a1..f382992cb0a 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -26,7 +26,8 @@ module Gitlab stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, - duration: pipeline.duration + duration: pipeline.duration, + variables: pipeline.variables.map(&:hook_attrs) } end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index c169c8fe135..b498f113859 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -97,11 +97,15 @@ module Gitlab } end - # This method provide a sample data generated with + # This method provides a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) + # Use sample data if repo has no commit + # (expect the case of test service configuration settings) + return sample_data if project.empty_repo? + ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + commits = project.repository.commits(project.default_branch.to_s, limit: 3) build(project, user, commits.last&.id, commits.first&.id, ref, commits) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8eacad078c8..42f9605f5ac 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -249,5 +249,21 @@ module Gitlab end private_class_method :database_version + + def self.add_post_migrate_path_to_rails(force: false) + return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force + + Rails.application.config.paths['db'].each do |db_path| + path = Rails.root.join(db_path, 'post_migrate').to_s + + unless Rails.application.config.paths['db/migrate'].include? path + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path + end + end + end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index d32837f5793..7d334a79009 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -2,6 +2,8 @@ module Gitlab module Database # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base + include FromUnion + self.table_name = if Database.postgresql? 'information_schema.role_table_grants' @@ -42,9 +44,7 @@ module Gitlab .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") ] - union = SQL::Union.new(queries).to_sql - - Grant.from("(#{union}) privs").any? + Grant.from_union(queries, alias_as: 'privs').any? end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7f012312819..30541ee3553 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1073,6 +1073,10 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + + def mysql_compatible_index_length + Gitlab::Database.mysql? ? 20 : nil + end end end end diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb new file mode 100644 index 00000000000..36e4559b554 --- /dev/null +++ b/lib/gitlab/database/subquery.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Subquery + class << self + def self_join(relation) + t = relation.arel_table + t2 = if !Gitlab.rails5? + relation.arel.as('t2') + else + # Work around a bug in Rails 5, where LIMIT causes trouble + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 + r = relation.limit(nil).arel + r.take(relation.limit_value) if relation.limit_value + r.as('t2') + end + + relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d16a55720b7..fb117baca9e 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -20,8 +20,9 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil) @diff = diff + @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs @@ -165,11 +166,11 @@ module Gitlab end def added_lines - diff_lines.count(&:added?) + @stats&.additions || diff_lines.count(&:added?) end def removed_lines - diff_lines.count(&:removed?) + @stats&.deletions || diff_lines.count(&:removed?) end def file_identifier @@ -211,13 +212,17 @@ module Gitlab old_blob && new_blob && old_blob.binary? != new_blob.binary? end + # rubocop: disable CodeReuse/ActiveRecord def size valid_blobs.map(&:size).sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_size valid_blobs.map(&:raw_size).sum end + # rubocop: enable CodeReuse/ActiveRecord def raw_binary? try_blobs(:raw_binary?) diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index c79d8d3cb21..b79ff771a2b 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,23 +2,27 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) @diffable = diffable + @include_stats = diff_options.delete(:include_stats) @diffs = diffable.raw_diffs(diff_options) @project = project @diff_options = diff_options @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @repository = project.repository end def diff_files @@ -33,12 +37,37 @@ module Gitlab diff_files.find { |diff_file| diff_file.new_path == new_path } end + def clear_cache + # No-op + end + + def write_cache + # No-op + end + private + def diff_stats_collection + strong_memoize(:diff_stats) do + # There are scenarios where we don't need to request Diff Stats, + # when caching for instance. + next unless @include_stats + next unless diff_refs + + @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha) + end + end + def decorate_diff!(diff) return diff if diff.is_a?(File) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) + stats = diff_stats_collection&.find_by_path(diff.new_path) + + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: diff_refs, + fallback_diff_refs: fallback_diff_refs, + stats: stats) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index be25e1bab21..0dd073a3a8e 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -2,6 +2,8 @@ module Gitlab module Diff module FileCollection class MergeRequestDiff < Base + extend ::Gitlab::Utils::Override + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -13,70 +15,35 @@ module Gitlab end def diff_files - # Make sure to _not_ send any method call to Gitlab::Diff::File - # _before_ all of them were collected (`super`). Premature method calls will - # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy). - # diff_files = super - diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) } - store_highlight_cache + diff_files.each { |diff_file| cache.decorate(diff_file) } diff_files end - def real_size - @merge_request_diff.real_size + override :write_cache + def write_cache + cache.write_if_empty end - def clear_cache! - Rails.cache.delete(cache_key) + override :clear_cache + def clear_cache + cache.clear end def cache_key - [@merge_request_diff, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] - end - - private - - def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) - diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| - Gitlab::Diff::Line.init_from_hash(line) - end + cache.key end - # - # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted) - # for the highlighted ones, so we just skip their execution. - # If the highlighted diff files lines are not cached we calculate and cache them. - # - # The content of the cache is a Hash where the key identifies the file and the values are Arrays of - # hashes that represent serialized diff lines. - # - def cache_highlight!(diff_file) - item_key = diff_file.file_identifier - - if highlight_cache[item_key] - highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) - else - highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) - end - end - - def highlight_cache - return @highlight_cache if defined?(@highlight_cache) - - @highlight_cache = Rails.cache.read(cache_key) || {} - @highlight_cache_was_empty = @highlight_cache.empty? - @highlight_cache + def real_size + @merge_request_diff.real_size end - def store_highlight_cache - Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty - end + private - def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.text? && diff_file.diffable? + def cache + @cache ||= Gitlab::Diff::HighlightCache.new(self) end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 1f012043e56..a605ddb5c33 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,7 +24,7 @@ module Gitlab # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || diff_line.text + rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) if line_inline_diffs = inline_diffs[i] begin diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb new file mode 100644 index 00000000000..e4390771db2 --- /dev/null +++ b/lib/gitlab/diff/highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class HighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 99970779c67..72d5ec547da 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -67,6 +67,7 @@ module Gitlab private # Finds pairs of old/new line pairs that represent the same line that changed + # rubocop: disable CodeReuse/ActiveRecord def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` @@ -89,6 +90,7 @@ module Gitlab changed_line_pairs end + # rubocop: enable CodeReuse/ActiveRecord end private diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 1ab6df0b6ae..5b67cd46c48 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -79,16 +79,10 @@ module Gitlab } end + # We have to keep this here since it is still used for conflict resolution + # Conflict::File#as_json renders json diff lines in sections def as_json(opts = nil) - { - line_code: line_code, - type: type, - old_line: old_line, - new_line: new_line, - text: text, - rich_text: rich_text || CGI.escapeHTML(text), - meta_data: meta_positions - } + DiffLineSerializer.new.represent(self) end private diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 978962ab2eb..f967494199e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -69,6 +69,10 @@ module Gitlab JSON.generate(formatter.to_h, opts) end + def as_json(opts = nil) + to_h.as_json(opts) + end + def type formatter.line_age end @@ -101,21 +105,21 @@ module Gitlab return @diff_file if defined?(@diff_file) @diff_file = begin - if RequestStore.active? - key = { - project_id: repository.project.id, - start_sha: start_sha, - head_sha: head_sha, - path: file_path - } - - RequestStore.fetch(key) { find_diff_file(repository) } - else - find_diff_file(repository) - end + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) } end end + def diff_options + { paths: paths, expanded: true, include_stats: false } + end + def diff_line(repository) @diff_line ||= diff_file(repository)&.line_for_position(self) end @@ -130,7 +134,7 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(paths: paths, expanded: true).diff_files.first + comparison.diffs(diff_options).diff_files.first end def get_formatter_class(type) diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index e08b5be8984..cebedb19dcc 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,20 +1,23 @@ -require 'gitlab/email/handler/create_merge_request_handler' -require 'gitlab/email/handler/create_note_handler' -require 'gitlab/email/handler/create_issue_handler' -require 'gitlab/email/handler/unsubscribe_handler' +# frozen_string_literal: true module Gitlab module Email module Handler - HANDLERS = [ - UnsubscribeHandler, - CreateNoteHandler, - CreateMergeRequestHandler, - CreateIssueHandler - ].freeze + def self.handlers + @handlers ||= load_handlers + end + + def self.load_handlers + [ + UnsubscribeHandler, + CreateNoteHandler, + CreateMergeRequestHandler, + CreateIssueHandler + ] + end def self.for(mail, mail_key) - HANDLERS.find do |klass| + handlers.find do |klass| handler = klass.new(mail, mail_key) break handler if handler.can_handle? end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 0bba433d04b..35bb49ad19a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index fc8615afcae..69982efbbe6 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' module Gitlab @@ -28,9 +30,11 @@ module Gitlab record_name: 'issue') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end + # rubocop: enable CodeReuse/ActiveRecord def project @project ||= Project.find_by_full_path(project_path) diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 2316e58c3fc..e68ae60ff98 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' @@ -32,9 +34,11 @@ module Gitlab record_name: 'merge_request') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end + # rubocop: enable CodeReuse/ActiveRecord def project @project ||= Project.find_by_full_path(project_path) diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 379b114e957..c7c573595fa 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 38b1425364f..ff6b2c729b2 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 56751e4e41e..d2f617b868a 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' module Gitlab diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bb14a8cd9e7..2c827265d8c 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -5,12 +5,6 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct - def initialize(options = {}) - super - - FakeApplicationSettings.define_predicate_methods(options) - end - # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -23,5 +17,23 @@ module Gitlab end end end + + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + def key_restriction_for(type) + 0 + end + + def allowed_key_types + ApplicationSetting::SUPPORTED_KEY_TYPES + end + + def pick_repository_storage + repository_storages.sample + end end end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 4850a6c0430..050a1ad3a0b 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -47,7 +47,7 @@ module Gitlab end def appearance - RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) + Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new) end def appearance_favicon diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 49bc9c0b671..8f55e94975c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab # Project files readme: %r{\Areadme[^/]*\z}i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, - license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb new file mode 100644 index 00000000000..5386656efe7 --- /dev/null +++ b/lib/gitlab/file_markdown_link_builder.rb @@ -0,0 +1,21 @@ +# Builds the markdown link of a file +# It needs the methods filename and secure_url (final destination url) to be defined. +module Gitlab + module FileMarkdownLinkBuilder + include FileTypeDetection + + def markdown_link + return unless name = markdown_name + + markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})" + markdown.prepend("!") if image_or_video? || dangerous? + markdown + end + + def markdown_name + return unless filename.present? + + image_or_video? ? File.basename(filename, File.extname(filename)) : filename + end + end +end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb new file mode 100644 index 00000000000..25ee07cf940 --- /dev/null +++ b/lib/gitlab/file_type_detection.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# File helpers methods. +# It needs the method filename to be defined. +module Gitlab + module FileTypeDetection + IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + # We recommend using the .mp4 format over .mov. Videos in .mov format can + # still be used but you really need to make sure they are served with the + # proper MIME type video/mp4 and not video/quicktime or your videos won't play + # on IE >= 9. + # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html + VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze + # These extension types can contain dangerous code and should only be embedded inline with + # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline". + DANGEROUS_EXT = %w[svg].freeze + + def image? + extension_match?(IMAGE_EXT) + end + + def video? + extension_match?(VIDEO_EXT) + end + + def image_or_video? + image? || video? + end + + def dangerous? + extension_match?(DANGEROUS_EXT) + end + + private + + def extension_match?(extensions) + return false unless filename + + extension = File.extname(filename).delete('.') + extensions.include?(extension.downcase) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index a91de278cf3..98ea5b309a1 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -79,6 +79,7 @@ module Gitlab ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end + # rubocop: disable CodeReuse/ActiveRecord def user_info(person_id) user_hash = user_map[person_id.to_s] @@ -95,7 +96,9 @@ module Gitlab { name: user_name, gitlab_id: gitlab_id } end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def import_cases return unless @cases @@ -141,6 +144,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def opened_content(comments) while comment = comments.shift diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5b264868af0..74cdabfed9d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -53,9 +53,6 @@ module Gitlab # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - # A rugged reference? - commit_id = Gitlab::Git::Ref.dereference_object(commit_id) - # Some weird thing? return nil unless commit_id.is_a?(String) @@ -127,8 +124,6 @@ module Gitlab # :topo, or any combination of them (in an array). Commit ordering types # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) repo.wrapped_gitaly_errors do Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) @@ -328,7 +323,6 @@ module Gitlab entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) return unless entry - # To be compatible with the rugged format entry = entry.to_h entry.delete(:data) entry[:name] = File.basename(path) @@ -346,8 +340,8 @@ module Gitlab subject: message_split[0] ? message_split[0].chomp.b : "", body: raw_commit.message.b, parent_ids: raw_commit.parent_ids, - author: gitaly_commit_author_from_rugged(raw_commit.author), - committer: gitaly_commit_author_from_rugged(raw_commit.committer) + author: gitaly_commit_author_from_raw(raw_commit.author), + committer: gitaly_commit_author_from_raw(raw_commit.committer) ) end @@ -381,7 +375,7 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_commit_author_from_rugged(author_or_committer) + def gitaly_commit_author_from_raw(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, email: author_or_committer[:email].b, diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb deleted file mode 100644 index 4198be7c9c9..00000000000 --- a/lib/gitlab/git/committer_with_hooks.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Git - class CommitterWithHooks < Gollum::Committer - attr_reader :gl_wiki - - def initialize(gl_wiki, options = {}) - @gl_wiki = gl_wiki - super(gl_wiki.gollum_wiki, options) - end - - def commit - # TODO: Remove after 10.8 - return super unless allowed_to_run_hooks? - - result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( - @wiki.ref, - start_branch_name: @wiki.ref - ) do |start_commit| - super(false) - end - - result[:newrev] - rescue Gitlab::Git::PreReceiveError => e - message = "Custom Hook failed: #{e.message}" - raise Gitlab::Git::Wiki::OperationError, message - end - - private - - # TODO: Remove after 10.8 - def allowed_to_run_hooks? - @options[:user_id] != 0 && @options[:username].present? - end - - def git_user - @git_user ||= Gitlab::Git::User.new(@options[:username], - @options[:name], - @options[:email], - gitlab_id) - end - - def gitlab_id - Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) - end - end - end -end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 61ce10ca131..0d96211f4d4 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,6 +1,3 @@ -# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. - -# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git class Diff @@ -22,13 +19,17 @@ module Gitlab alias_method :expanded?, :expanded - SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + # The default maximum content size to display a diff patch. + # + # If this value ever changes, make sure to create a migration to update + # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. + DEFAULT_MAX_PATCH_BYTES = 100.kilobytes - # The maximum size of a diff to display. - SIZE_LIMIT = 100.kilobytes + # This is a limitation applied on the source (Gitaly), therefore we don't allow + # persisting limits over that. + MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - # The maximum size before a diff is collapsed. - COLLAPSE_LIMIT = 10.kilobytes + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self def between(repo, head, base, options = {}, *paths) @@ -52,20 +53,31 @@ module Gitlab repo.diff(common_commit, head, actual_options, *paths) end - # Return a copy of the +options+ hash containing only keys that can be - # passed to Rugged. Allowed options are: + # Return a copy of the +options+ hash containing only recognized keys. + # Allowed options are: # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :disable_pathspec_match :: - # If true, the given +*paths+ will be applied as exact matches, - # instead of as fnmatch patterns. + # :max_files :: + # Limit how many files will patches be allowed for before collapsing + # + # :max_lines :: + # Limit how many patch lines (across all files) will be allowed for + # before collapsing + # + # :limits :: + # A hash with additional limits to check before collapsing patches. + # Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines` + # and `safe_max_bytes` # + # :expanded :: + # If true, patch raw data will not be included in the diff after + # `max_files`, `max_lines` or any of the limits in `limits` are + # exceeded def filter_diff_options(options, default_options = {}) - allowed_options = [:ignore_whitespace_change, - :disable_pathspec_match, :paths, - :max_files, :max_lines, :limits, :expanded] + allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, + :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -93,10 +105,30 @@ module Gitlab # # "Binary files a/file/path and b/file/path differ\n" # This is used when we detect that a diff is binary - # using CharlockHolmes when Rugged treats it as text. + # using CharlockHolmes. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end + + # Returns the limit of bytes a single diff file can reach before it + # appears as 'collapsed' for end-users. + # By convention, it's 10% of the persisted `diff_max_patch_bytes`. + # + # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by + # default. + # + # Patches surpassing this limit should still be persisted in the database. + def patch_safe_limit_bytes + patch_hard_limit_bytes / 10 + end + + # Returns the limit for a single diff file (patch). + # + # Patches surpassing this limit shouldn't be persisted in the database + # and will be presented as 'too large' for end-users. + def patch_hard_limit_bytes + Gitlab::CurrentSettings.diff_max_patch_bytes + end end def initialize(raw_diff, expanded: true) @@ -106,8 +138,6 @@ module Gitlab when Hash init_from_hash(raw_diff) prune_diff_if_eligible - when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff) when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible @@ -144,7 +174,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -162,7 +192,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT + @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes end def collapse! @@ -184,31 +214,6 @@ module Gitlab private - def init_from_rugged(rugged) - if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged) - d = rugged.delta - else - d = rugged - end - - @new_path = encode!(d.new_file[:path]) - @old_path = encode!(d.old_file[:path]) - @a_mode = d.old_file[:mode].to_s(8) - @b_mode = d.new_file[:mode].to_s(8) - @new_file = d.added? - @renamed_file = d.renamed? - @deleted_file = d.deleted? - end - - def init_from_rugged_patch(patch) - # Don't bother initializing diffs that are too large. If a diff is - # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch) - - @diff = encode!(strip_diff_headers(patch.to_s)) - end - def init_from_hash(hash) raw_diff = hash.symbolize_keys @@ -238,47 +243,6 @@ module Gitlab collapse! end end - - # If the patch surpasses any of the diff limits it calls the appropiate - # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch) - size = 0 - - patch.each_hunk do |hunk| - hunk.each_line do |line| - size += line.content.bytesize - - if size >= SIZE_LIMIT - too_large! - return true # rubocop:disable Cop/AvoidReturnFromBlocks - end - end - end - - if !expanded && size >= COLLAPSE_LIMIT - collapse! - return true - end - - false - end - - # Strip out the information at the beginning of the patch's text to match - # Grit's output - def strip_diff_headers(diff_text) - # Delete everything up to the first line that starts with '---' or - # 'Binary' - diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') - - if diff_text.start_with?('---', 'Binary') - diff_text - else - # If the diff_text did not contain a line starting with '---' or - # 'Binary', return the empty string. No idea why; we are just - # preserving behavior from before the refactor. - '' - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 219c69893ad..47ebca7c4a2 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,7 +11,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.collection_limits(options = {}) + def self.limits(options = {}) limits = {} limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) @@ -19,13 +19,14 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes OpenStruct.new(limits) end def initialize(iterator, options = {}) @iterator = iterator - @limits = self.class.collection_limits(options) + @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb new file mode 100644 index 00000000000..998c41497a2 --- /dev/null +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class DiffStatsCollection + include Gitlab::Utils::StrongMemoize + include Enumerable + + def initialize(diff_stats) + @collection = diff_stats + end + + def each(&block) + @collection.each(&block) + end + + def find_by_path(path) + indexed_by_path[path] + end + + def paths + @collection.map(&:path) + end + + private + + def indexed_by_path + strong_memoize(:indexed_by_path) do + index_by { |stats| stats.path } + end + end + end + end +end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 5ff15a787f0..00000000000 --- a/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,253 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - git_import_repository(source, timeout) - end - - def fork_repository(new_shard_name, new_repository_relative_path) - git_fork_repository(new_shard_name, new_repository_relative_path) - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = fetch_remote_command(name, tags, prune, force) - - setup_ssh_auth(ssh_key, known_hosts) do |env| - run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - end - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def fetch_remote_command(name, tags, prune, force) - %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| - cmd << '--prune' if prune - cmd << '--force' if force - cmd << (tags ? '--tags' : '--no-tags') - end - end - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - end - end -end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb deleted file mode 100644 index 94ff5b4980a..00000000000 --- a/lib/gitlab/git/hook.rb +++ /dev/null @@ -1,108 +0,0 @@ -# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in -# app/services. We shouldn't bother migrating this until we know how -# Gitlab::Git::HooksService will be migrated. - -module Gitlab - module Git - class Hook - GL_PROTOCOL = 'web'.freeze - attr_reader :name, :path, :repository - - def initialize(name, repository) - @name = name - @repository = repository - @path = File.join(repo_path, 'hooks', name) - end - - def repo_path - repository.path - end - - def exists? - File.exist?(path) - end - - def trigger(gl_id, gl_username, oldrev, newrev, ref) - return [true, nil] unless exists? - - Bundler.with_clean_env do - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - end - end - end - - private - - def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - changes = [oldrev, newrev, ref].join(" ") - - exit_status = false - exit_message = nil - - vars = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => repository.gl_repository - } - - options = { - chdir: repo_path - } - - Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| - exit_status = true - stdin.sync = true - - # in git, pre- and post- receive hooks may just exit without - # reading stdin. We catch the exception to avoid a broken pipe - # warning - begin - # inject all the changes as stdin to the hook - changes.lines do |line| - stdin.puts line - end - rescue Errno::EPIPE - end - - stdin.close - - unless wait_thr.value == 0 - exit_status = false - exit_message = retrieve_error_message(stderr, stdout) - end - end - - [exit_status, exit_message] - end - - def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path - } - - options = { - chdir: repo_path - } - - args = [ref, oldrev, newrev] - - stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, stderr.presence || stdout] - end - - def retrieve_error_message(stderr, stdout) - err_message = stderr.read - err_message = err_message.blank? ? stdout.read : err_message - err_message - end - end - end -end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index 455e8451c10..620568d8817 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -17,18 +17,18 @@ module Gitlab ].freeze def self.set(gl_repository, env) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? raise "missing gl_repository" if gl_repository.blank? - RequestStore.store[:gitlab_git_env] ||= {} - RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end def self.all(gl_repository) - return {} unless RequestStore.active? + return {} unless Gitlab::SafeRequestStore.active? - h = RequestStore.fetch(:gitlab_git_env) { {} } + h = Gitlab::SafeRequestStore.fetch(:gitlab_git_env) { {} } h.fetch(gl_repository, {}) end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb deleted file mode 100644 index e67cacdb95a..00000000000 --- a/lib/gitlab/git/hooks_service.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module Git - class HooksService - attr_accessor :oldrev, :newrev, :ref - - def execute(pusher, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = pusher.gl_id - @gl_username = pusher.username - @oldrev = oldrev - @newrev = newrev - @ref = ref - - %w(pre-receive update).each do |hook_name| - status, message = run_hook(hook_name) - - unless status - raise PreReceiveError, message - end - end - - yield(self).tap do - run_hook('post-receive') - end - end - - private - - def run_hook(name) - hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) - end - end - end -end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index d94082a3e30..c2e4274e3ee 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,157 +1,7 @@ -# Gitaly note: JV: When the time comes I think we will want to copy this -# class into Gitaly. None of its methods look like they should be RPC's. -# The RPC's will be at a higher level. - module Gitlab module Git class Index IndexError = Class.new(StandardError) - - DEFAULT_MODE = 0o100644 - - ACTIONS = %w(create create_dir update move delete).freeze - ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze - - attr_reader :repository, :raw_index - - def initialize(repository) - @repository = repository - @raw_index = repository.rugged.index - end - - delegate :read_tree, :get, to: :raw_index - - def apply(action, options) - validate_action!(action) - public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend - end - - def write_tree - raw_index.write_tree(repository.rugged) - end - - def dir_exists?(path) - raw_index.find { |entry| entry[:path].start_with?("#{path}/") } - end - - def create(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - add_blob(options) - end - - def create_dir(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - if dir_exists?(options[:file_path]) - raise IndexError, "A directory with this name already exists" - end - - options = options.dup - options[:file_path] += '/.gitkeep' - options[:content] = '' - - add_blob(options) - end - - def update(options) - options = normalize_options(options) - - file_entry = get(options[:file_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - add_blob(options, mode: file_entry[:mode]) - end - - def move(options) - options = normalize_options(options) - - file_entry = get(options[:previous_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - raw_index.remove(options[:previous_path]) - - add_blob(options, mode: file_entry[:mode]) - end - - def delete(options) - options = normalize_options(options) - - unless get(options[:file_path]) - raise IndexError, "A file with this name doesn't exist" - end - - raw_index.remove(options[:file_path]) - end - - private - - def normalize_options(options) - options = options.dup - options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] - options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] - options - end - - def normalize_path(path) - unless path - raise IndexError, "You must provide a file path" - end - - pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - - pathname.each_filename do |segment| - if segment == '..' - raise IndexError, 'Path cannot include directory traversal' - end - end - - pathname.to_s - end - - def add_blob(options, mode: nil) - content = options[:content] - unless content - raise IndexError, "You must provide content" - end - - content = Base64.decode64(content) if options[:encoding] == 'base64' - - detect = CharlockHolmes::EncodingDetector.new.detect(content) - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repository.rugged.write(content, :blob) - - raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) - rescue Rugged::IndexError => e - raise IndexError, e.message - end - - def validate_action!(action) - unless ACTIONS.include?(action.to_s) - raise ArgumentError, "Unknown action '#{action}'" - end - end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 57d748343be..0584629ac84 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,8 +1,6 @@ module Gitlab module Git class OperationService - include Gitlab::Git::Popen - BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created @@ -17,177 +15,6 @@ module Gitlab ) end end - - attr_reader :user, :repository - - def initialize(user, new_repository) - if user - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - @user = user - end - - # Refactoring aid - Gitlab::Git.check_namespace!(new_repository) - - @repository = new_repository - end - - def add_branch(branch_name, newrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - oldrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def rm_branch(branch) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name - oldrev = branch.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def add_tag(tag_name, newrev, options = {}) - ref = Gitlab::Git::TAG_REF_PREFIX + tag_name - oldrev = Gitlab::Git::BLANK_SHA - - with_hooks(ref, newrev, oldrev) do |service| - # We want to pass the OID of the tag object to the hooks. For an - # annotated tag we don't know that OID until after the tag object - # (raw_tag) is created in the repository. That is why we have to - # update the value after creating the tag object. Only the - # "post-receive" hook will receive the correct value in this case. - raw_tag = repository.rugged.tags.create(tag_name, newrev, options) - service.newrev = raw_tag.target_id - end - end - - def rm_tag(tag) - ref = Gitlab::Git::TAG_REF_PREFIX + tag.name - oldrev = tag.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) do - repository.rugged.tags.delete(tag_name) - end - end - - # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, - # it would be created from `start_branch_name`. - # If `start_repository` is passed, and the branch doesn't exist, - # it would try to find the commits from it instead of current repository. - def with_branch( - branch_name, - start_branch_name: nil, - start_repository: repository, - &block) - - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - start_branch_name = nil if start_repository.empty? - - if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" - end - - update_branch_with_hooks(branch_name) do - repository.with_repo_branch_commit( - start_repository, - start_branch_name || branch_name, - &block) - end - end - - def update_branch(branch_name, newrev, oldrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - end - - private - - # Returns [newrev, should_run_after_create, should_run_after_create_branch] - def update_branch_with_hooks(branch_name) - update_autocrlf_option - - was_empty = repository.empty? - - # Make commit - newrev = yield - - unless newrev - raise Gitlab::Git::CommitError.new('Failed to create commit') - end - - branch = repository.find_branch(branch_name) - oldrev = find_oldrev_from_branch(newrev, branch) - - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - - BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) - end - - def find_oldrev_from_branch(newrev, branch) - return Gitlab::Git::BLANK_SHA unless branch - - oldrev = branch.target - - merge_base = repository.merge_base(newrev, branch.target) - raise Gitlab::Git::Repository::InvalidRef unless merge_base - - if oldrev == merge_base - oldrev - else - raise Gitlab::Git::CommitError.new('Branch diverged') - end - end - - def update_ref_in_hooks(ref, newrev, oldrev) - with_hooks(ref, newrev, oldrev) do - update_ref(ref, newrev, oldrev) - end - end - - def with_hooks(ref, newrev, oldrev) - Gitlab::Git::HooksService.new.execute( - user, - repository, - oldrev, - newrev, - ref) do |service| - - yield(service) - end - end - - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. - def update_ref(ref, newrev, oldrev) - # We use 'git update-ref' because libgit2/rugged currently does not - # offer 'compare and swap' ref updates. Without compare-and-swap we can - # (and have!) accidentally reset the ref to an earlier state, clobbering - # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - - output, status = popen( - command, - repository.path) do |stdin| - stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") - end - - unless status.zero? - Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") - raise Gitlab::Git::CommitError.new( - "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ - " Please refresh and try again.") - end - end - - def update_autocrlf_option - if repository.autocrlf != :input - repository.autocrlf = :input - end - end end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb deleted file mode 100644 index 7426688fc55..00000000000 --- a/lib/gitlab/git/popen.rb +++ /dev/null @@ -1,112 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -require 'open3' - -module Gitlab - module Git - module Popen - FAST_GIT_PROCESS_TIMEOUT = 15.seconds - - def popen(cmd, path, vars = {}, lazy_block: nil) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - options = { chdir: path } - - cmd_output = "" - cmd_status = 0 - Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - stdout.set_encoding(Encoding::ASCII_8BIT) - - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - err_reader = Thread.new { stderr.read } - - yield(stdin) if block_given? - stdin.close - - if lazy_block - cmd_output = lazy_block.call(stdout.lazy) - cmd_status = 0 - break - else - cmd_output << stdout.read - end - - cmd_output << err_reader.value - cmd_status = wait_thr.value.exitstatus - end - - [cmd_output, cmd_status] - end - - def popen_with_timeout(cmd, timeout, path, vars = {}) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - - unless File.directory?(path) - FileUtils.mkdir_p(path) - end - - rout, wout = IO.pipe - rerr, werr = IO.pipe - - pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - out_reader = Thread.new { rout.read } - err_reader = Thread.new { rerr.read } - - begin - # close write ends so we could read them - wout.close - werr.close - - status = process_wait_with_timeout(pid, timeout) - - cmd_output = out_reader.value - cmd_output << err_reader.value # Copying the behaviour of `popen` which merges stderr into output - - [cmd_output, status.exitstatus] - rescue Timeout::Error => e - kill_process_group_for_pid(pid) - - raise e - ensure - wout.close unless wout.closed? - werr.close unless werr.closed? - - rout.close - rerr.close - end - end - - def process_wait_with_timeout(pid, timeout) - deadline = timeout.seconds.from_now - wait_time = 0.01 - - while deadline > Time.now - sleep(wait_time) - _, status = Process.wait2(pid, Process::WNOHANG) - - return status unless status.nil? - end - - raise Timeout::Error, "Timeout waiting for process ##{pid}" - end - - def kill_process_group_for_pid(pid) - Process.kill("KILL", -pid) - Process.wait(pid) - rescue Errno::ESRCH - end - end - end -end diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb new file mode 100644 index 00000000000..b6577ba17f1 --- /dev/null +++ b/lib/gitlab/git/push.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class Push + include Gitlab::Utils::StrongMemoize + + attr_reader :ref, :oldrev, :newrev + + def initialize(project, oldrev, newrev, ref) + @project = project + @oldrev = oldrev.presence || Gitlab::Git::BLANK_SHA + @newrev = newrev.presence || Gitlab::Git::BLANK_SHA + @ref = ref + end + + def branch_name + strong_memoize(:branch_name) do + Gitlab::Git.branch_name(@ref) + end + end + + def branch_added? + Gitlab::Git.blank_ref?(@oldrev) + end + + def branch_removed? + Gitlab::Git.blank_ref?(@newrev) + end + + def branch_updated? + branch_push? && !branch_added? && !branch_removed? + end + + def force_push? + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def branch_push? + strong_memoize(:branch_push) do + Gitlab::Git.branch_ref?(@ref) + end + end + + def modified_paths + unless branch_updated? + raise ArgumentError, 'Unable to calculate modified paths!' + end + + strong_memoize(:modified_paths) do + @project.repository.diff_stats(@oldrev, @newrev).paths + end + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index fa71a4e7ea7..31a280155bd 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). - module Gitlab module Git class Ref @@ -26,13 +24,6 @@ module Gitlab str.gsub(%r{\Arefs/heads/}, '') end - # Gitaly: this method will probably be migrated indirectly via its call sites. - def self.dereference_object(object) - object = object.target while object.is_a?(Rugged::Tag::Annotation) - - object - end - def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) @dereferenced_target = dereferenced_target diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9521a2d63a0..7732049b69b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,18 +6,9 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize - ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY - GIT_ALTERNATE_OBJECT_DIRECTORIES - ].freeze - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 @@ -73,7 +64,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -82,13 +73,6 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - @name = @relative_path.split("/").last end @@ -112,19 +96,6 @@ module Gitlab raise Gitlab::Git::CommandError.new(e.message) end - # This method will be removed when Gitaly reaches v1.1. - def rugged - circuit_breaker.perform do - Rugged::Repository.new(path, alternates: alternate_object_directories) - end - rescue Rugged::RepositoryError, Rugged::OSError - raise NoRepository.new('no repository for such path') - end - - def cleanup - @rugged&.close - end - def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -148,10 +119,6 @@ module Gitlab end end - def reload_rugged - @rugged = nil - end - # Directly find a branch with a simple name (e.g. master) # def find_branch(name) @@ -250,15 +217,6 @@ module Gitlab end end - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -331,7 +289,7 @@ module Gitlab (size.to_f / 1024).round(2) end - # Use the Rugged Walker API to build an array of commits. + # Build an array of commits. # # Usage. # repo.log( @@ -463,6 +421,16 @@ module Gitlab Gitlab::Git::DiffCollection.new(iterator, options) end + def diff_stats(left_id, right_id) + stats = wrapped_gitaly_errors do + gitaly_commit_client.diff_stats(left_id, right_id) + end + + Gitlab::Git::DiffStatsCollection.new(stats) + rescue CommandError, TypeError + Gitlab::Git::DiffStatsCollection.new([]) + end + # Returns a RefName for a given SHA def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? @@ -591,19 +559,6 @@ module Gitlab end end - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) args = { user: user, @@ -619,14 +574,6 @@ module Gitlab end end - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end - - def user_to_committer(user) - Gitlab::Git.committer_hash(email: user.email, name: user.name) - end - # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do @@ -666,18 +613,12 @@ module Gitlab end end - AUTOCRLF_VALUES = { - "true" => true, - "false" => false, - "input" => :input - }.freeze + def find_remote_root_ref(remote_name) + return unless remote_name.present? - def autocrlf - AUTOCRLF_VALUES[rugged.config['core.autocrlf']] - end - - def autocrlf=(value) - rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + wrapped_gitaly_errors do + gitaly_remote_client.find_remote_root_ref(remote_name) + end end # Returns result like "git ls-files" , recursive and full file path @@ -738,48 +679,6 @@ module Gitlab end end - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end - end - end - - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name - - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end - - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" - ) - - yield commit(sha) - ensure - delete_refs(tmp_ref) if tmp_ref - end - def fetch_source_branch!(source_repository, source_branch, local_ref) wrapped_gitaly_errors do gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) @@ -809,21 +708,6 @@ module Gitlab end end - # This method, fetch_ref, is used from within - # Gitlab::Git::OperationService. OperationService will eventually only - # exist in gitaly-ruby. When we delete OperationService from gitlab-ce - # we can also remove fetch_ref. - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - message, status = run_git(args, env: source_repository.fetch_env) - raise Gitlab::Git::CommandError, message if status != 0 - - target_ref - end - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) @@ -891,24 +775,6 @@ module Gitlab end end - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - def bundle_to_disk(save_path) wrapped_gitaly_errors do gitaly_repository_client.create_bundle(save_path) @@ -1056,9 +922,10 @@ module Gitlab end end - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) + wrapped_gitaly_errors do + gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) + end end def last_commit_for_path(sha, path) @@ -1067,26 +934,6 @@ module Gitlab end end - def rev_list(including: [], excluding: [], options: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) - end - - args.push('--objects') if objects - - if options.any? - args.push(*options) - end - - run_git!(args, lazy_block: block) - end - def checksum # The exists? RPC is much cheaper, so we perform this request first raise NoRepository, "Repository does not exists" unless exists? @@ -1104,44 +951,6 @@ module Gitlab end end - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - def gitaly_merged_branch_names(branch_names, root_sha) qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } @@ -1184,31 +993,6 @@ module Gitlab found_module && found_module['url'] end - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -1223,14 +1007,6 @@ module Gitlab def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end end end end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 62427ac9cc4..fcee9ae566c 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -11,7 +11,7 @@ module Gitlab to: :failure_info def self.for_storage(storage) - cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do + cached_circuitbreakers = Gitlab::SafeRequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| hash[storage_name] = build(storage_name) end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb index 387279c110d..1d28a850049 100644 --- a/lib/gitlab/git/storage/failure_info.rb +++ b/lib/gitlab/git/storage/failure_info.rb @@ -10,7 +10,7 @@ module Gitlab redis.del(*all_storage_keys) unless all_storage_keys.empty? end - RequestStore.delete(:circuitbreaker_cache) + Gitlab::SafeRequestStore.delete(:circuitbreaker_cache) end def self.load(cache_key) diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 90bbe85fd37..8e14acb4ccb 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -81,9 +81,11 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def total_failures @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index cb851b76a23..e0867aeb5a7 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -50,51 +50,6 @@ module Gitlab entry[:oid] end end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end end def initialize(options) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e573cd0e143..338e1a30c45 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user)) end def self.from_gitaly(gitaly_user) diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 1e14e8b652a..4bd91898457 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -1,8 +1,6 @@ module Gitlab module Git module Version - extend Gitlab::Git::Popen - def self.git_version Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 9d992be66eb..072019dfb0a 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -1,9 +1,16 @@ +# We only need Gollum::Page so let's not load all of gollum-lib. +require 'gollum-lib/pagination' +require 'gollum-lib/wiki' +require 'gollum-lib/page' + module Gitlab module Git class Wiki DuplicatePageError = Class.new(StandardError) OperationError = Class.new(StandardError) + DEFAULT_PAGINATION = Kaminari.config.default_per_page + CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do def to_h { user_id: user_id, username: username, name: name, email: email, message: message } @@ -74,7 +81,7 @@ module Gitlab # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 # per page, but also fetches 20 if `limit` or `per_page` < 20. # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION versions[0..slice_bound] end @@ -104,30 +111,6 @@ module Gitlab private - def new_page(gollum_page) - Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) - end - - def new_version(gollum_page, commit_id) - Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) - end - - def version(commit_id) - commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } - - if RequestStore.active? - RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } - else - commit_find_proc.call - end - end - - def assert_type!(object, klass) - unless object.is_a?(klass) - raise ArgumentError, "expected a #{klass}, got #{object.inspect}" - end - end - def gitaly_wiki_client @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end @@ -163,20 +146,6 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end - - def committer_with_hooks(commit_details) - Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) - end - - def with_committer_with_hooks(commit_details, &block) - committer = committer_with_hooks(commit_details) - - yield committer - - committer.commit - - nil - end end end end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index 84335aca4bc..64313bb04e8 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -3,17 +3,12 @@ module Gitlab class WikiFile attr_reader :mime_type, :raw_data, :name, :path - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::File is not serializable we must get all the data from - # 'gollum_file' during initialization, and NOT store it in an instance - # variable. - def initialize(gollum_file) - @mime_type = gollum_file.mime_type - @raw_data = gollum_file.raw_data - @name = gollum_file.name - @path = gollum_file.path + # This class wraps Gitlab::GitalyClient::WikiFile + def initialize(gitaly_file) + @mime_type = gitaly_file.mime_type + @raw_data = gitaly_file.raw_data + @name = gitaly_file.name + @path = gitaly_file.path end end end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index 669ae11a423..c4087c9ebdc 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -3,25 +3,15 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::Page is not serializable we must get all the data from - # 'gollum_page' during initialization, and NOT store it in an instance - # variable. - # - # Note that 'version' is a WikiPageVersion instance which it itself - # serializable. That means it's OK to store 'version' in an instance - # variable. - def initialize(gollum_page, version) - @url_path = gollum_page.url_path - @title = gollum_page.title - @format = gollum_page.format - @path = gollum_page.path - @raw_data = gollum_page.raw_data - @name = gollum_page.name - @historical = gollum_page.historical? - @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) + # This class abstracts away Gitlab::GitalyClient::WikiPage + def initialize(gitaly_page, version) + @url_path = gitaly_page.url_path + @title = gitaly_page.title + @format = gitaly_page.format + @path = gitaly_page.path + @raw_data = gitaly_page.raw_data + @name = gitaly_page.name + @historical = gitaly_page.historical? @version = version end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb index 55f1afedcab..d5e7e70fd31 100644 --- a/lib/gitlab/git/wiki_page_version.rb +++ b/lib/gitlab/git/wiki_page_version.rb @@ -3,11 +3,6 @@ module Gitlab class WikiPageVersion attr_reader :commit, :format - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are - # serializable. def initialize(commit, format) @commit = commit @format = format diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 35808149b90..240a0d7d1b8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -24,8 +24,8 @@ module Gitlab cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze - DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze - PUSH_COMMANDS = %w{ git-receive-pack }.freeze + DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze + PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes @@ -50,6 +50,10 @@ module Gitlab check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) + + custom_action = check_custom_action(cmd) + return custom_action if custom_action + check_db_accessibility!(cmd) ensure_project_on_push!(cmd, changes) @@ -65,7 +69,7 @@ module Gitlab check_push_access! end - true + ::Gitlab::GitAccessResult::Success.new end def guest_can_download_code? @@ -92,6 +96,10 @@ module Gitlab private + def check_custom_action(cmd) + nil + end + def check_valid_actor! return unless actor.is_a?(Key) diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb new file mode 100644 index 00000000000..a05a4baed82 --- /dev/null +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class CustomAction + attr_reader :payload, :message + + # Example of payload: + # + # { + # 'action' => 'geo_proxy_to_primary', + # 'data' => { + # 'api_endpoints' => %w{geo/proxy_git_push_ssh/info_refs geo/proxy_git_push_ssh/push}, + # 'gl_username' => user.username, + # 'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki) + # } + # } + # + def initialize(payload, message) + @payload = payload + @message = message + end + end + end +end diff --git a/lib/gitlab/git_access_result/success.rb b/lib/gitlab/git_access_result/success.rb new file mode 100644 index 00000000000..7bb9f24cb0e --- /dev/null +++ b/lib/gitlab/git_access_result/success.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class Success + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c27972a84a4..4ec87f6a3e7 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -174,10 +174,29 @@ module Gitlab end private_class_method :current_transaction_labels + # For some time related tasks we can't rely on `Time.now` since it will be + # affected by Timecop in some tests, and the clock of some gitaly-related + # components (grpc's c-core and gitaly server) use system time instead of + # timecop's time, so tests will fail. + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent + # timecop. + def self.real_time + Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + end + private_class_method :real_time + + def self.authorization_token(storage) + token = token(storage).to_s + issued_at = real_time.to_i.to_s + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + + "v2.#{hmac}.#{issued_at}" + end + private_class_method :authorization_token + def self.request_kwargs(storage, timeout, remote_storage: nil) - encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { - 'authorization' => "Bearer #{encoded_token}", + 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME } @@ -195,12 +214,7 @@ module Gitlab return result unless timeout > 0 - # Do not use `Time.now` for deadline calculation, since it - # will be affected by Timecop in some tests, but grpc's c-core - # uses system time instead of timecop's time, so tests will fail - # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will - # circumvent timecop - deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + deadline = real_time + timeout result[:deadline] = deadline result @@ -302,7 +316,7 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) # Only count limits in request-response environments (not sidekiq for example) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -326,7 +340,7 @@ module Gitlab end def self.allow_n_plus_1_calls - return yield unless RequestStore.active? + return yield unless Gitlab::SafeRequestStore.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -337,25 +351,25 @@ module Gitlab end def self.get_call_count(key) - RequestStore.store[key] || 0 + Gitlab::SafeRequestStore[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - RequestStore.store[key] ||= 0 - RequestStore.store[key] += 1 + Gitlab::SafeRequestStore[key] ||= 0 + Gitlab::SafeRequestStore[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - RequestStore.store[key] -= 1 + Gitlab::SafeRequestStore[key] -= 1 end private_class_method :decrement_call_count # Returns an estimate of the number of Gitaly calls made for this # request def self.get_request_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? gitaly_migrate_count = get_call_count("gitaly_migrate_actual") gitaly_call_count = get_call_count("gitaly_call_actual") @@ -372,28 +386,28 @@ module Gitlab end def self.reset_counts - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? %w[migrate call].each do |call_site| - RequestStore.store["gitaly_#{call_site}_actual"] = 0 - RequestStore.store["gitaly_#{call_site}_permitted"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_actual"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_permitted"] = 0 end end def self.add_call_details(details) id = details.delete(:id) - return unless id && RequestStore.active? && RequestStore.store[:peek_enabled] + return unless id && Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] ||= {} - RequestStore.store['gitaly_call_details'][id] ||= {} - RequestStore.store['gitaly_call_details'][id].merge!(details) + Gitlab::SafeRequestStore['gitaly_call_details'] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id].merge!(details) end def self.list_call_details - return {} unless RequestStore.active? && RequestStore.store[:peek_enabled] + return {} unless Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] || {} + Gitlab::SafeRequestStore['gitaly_call_details'] || {} end def self.expected_server_version @@ -431,22 +445,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") - RequestStore.store[:stack_counter] ||= Hash.new + Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new - count = RequestStore.store[:stack_counter][stack_string] || 0 - RequestStore.store[:stack_counter][stack_string] = count + 1 + count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 + Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -455,9 +469,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless RequestStore.active? + return nil unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return nil unless stack_counter max = max_call_count diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 6a97cd8ed17..085b2a127a5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -148,6 +148,24 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) + request = Gitaly::ListLastCommitsForTreeRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + path: encode_binary(path.to_s), + offset: offset, + limit: limit + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + + response.each_with_object({}) do |gitaly_response, hsh| + gitaly_response.commits.each do |commit_for_tree| + hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + end + end + end + def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, @@ -172,6 +190,17 @@ module Gitlab consume_commits_response(response) end + def diff_stats(left_commit_sha, right_commit_sha) + request = Gitaly::DiffStatsRequest.new( + repository: @gitaly_repo, + left_commit_id: left_commit_sha, + right_commit_id: right_commit_sha + ) + + response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response.flat_map(&:stats) + end + def find_all_commits(opts = {}) request = Gitaly::FindAllCommitsRequest.new( repository: @gitaly_repo, @@ -229,27 +258,29 @@ module Gitlab end def find_commit(revision) - if RequestStore.active? - # We don't use RequeStstore.fetch(key) { ... } directly because `revision` - # can be a branch name, so we can't use it as a key as it could point - # to another commit later on (happens a lot in tests). + if Gitlab::SafeRequestStore.active? + # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly + # because `revision` can be a branch name, so we can't use it as a key + # as it could point to another commit later on (happens a lot in + # tests). key = { storage: @gitaly_repo.storage_name, relative_path: @gitaly_repo.relative_path, commit_id: revision } - return RequestStore[key] if RequestStore.exist?(key) + return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) commit = call_find_commit(revision) return unless commit key[:commit_id] = commit.id - RequestStore[key] = commit + Gitlab::SafeRequestStore[key] = commit else call_find_commit(revision) end end + # rubocop: disable CodeReuse/ActiveRecord def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, @@ -259,6 +290,7 @@ module Gitlab response.sum(&:data) end + # rubocop: enable CodeReuse/ActiveRecord def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( @@ -369,7 +401,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 54c78fdb680..0f148614b20 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -333,7 +333,8 @@ module Gitlab action: action[:action].upcase.to_sym, file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), - base64_content: action[:encoding] == 'base64' + base64_content: action[:encoding] == 'base64', + execute_filemode: !!action[:execute_filemode] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 1381e033d4b..4661448621b 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RemoteService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def self.exists?(remote_url) @@ -52,6 +54,18 @@ module Gitlab response.result end + def find_remote_root_ref(remote_name) + request = Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + + response = GitalyClient.call(@storage, :remote_service, + :find_remote_root_ref, request) + + encode_utf8(response.ref) + end + def update_remote_mirror(ref_name, only_branches_matching) req_enum = Enumerator.new do |y| y.yield Gitaly::UpdateRemoteMirrorRequest.new( diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb index eb0e910665b..3a26dd58ff4 100644 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -5,6 +5,14 @@ module Gitlab @storage = storage end + # Returns all directories in the git storage directory, lexically ordered + def list_directories(depth: 1) + request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) + + GitalyClient.call(@storage, :storage_service, :list_directories, request) + .flat_map(&:paths) + end + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. def delete_all_repositories request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 8e530de174d..26d1f53f26c 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -13,7 +13,7 @@ module Gitlab Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. - If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`. + If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. MSG # This class will give easily recognizable NoMethodErrors diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 75be7d1f5a0..7c2c228ad01 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -110,7 +110,7 @@ module Gitlab repository: @gitaly_repo, page_path: encode_binary(page_path), page: options[:page] || 1, - per_page: options[:per_page] || Gollum::Page.per_page + per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION ) stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index a73033d35ba..80246fa1b77 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_labels = project.labels.pluck(:title).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Label, build_labels) diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 94eb9136b9a..8d54b27374c 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_milestones = project.milestones.pluck(:iid).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute # We insert records in bulk, by-passing any standard model callbacks. diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 100f459fdcc..0e7c9ee0d00 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_tags = project.releases.pluck(:tag).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Release, build_releases) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 01168abde6c..374dc9d3c00 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -14,11 +14,13 @@ module Gitlab end # Returns true if we should import the wiki for the project. + # rubocop: disable CodeReuse/ActiveRecord def import_wiki? client.repository(project.import_source)&.has_wiki && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end + # rubocop: enable CodeReuse/ActiveRecord # Imports the repository data. # diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index 9be071141db..d2479a8f565 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -18,6 +18,7 @@ module Gitlab Caching.read_integer(cache_key_for(name)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .labels @@ -28,6 +29,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(name) CACHE_KEY % { project: project.id, name: name } diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index 208d15dc144..5625730e796 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -21,6 +21,7 @@ module Gitlab Caching.read_integer(cache_key_for(issuable.milestone_number)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .milestones @@ -31,6 +32,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(iid) CACHE_KEY % { project: project.id, iid: iid } diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index be1259662a7..30283f147ef 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -136,13 +136,17 @@ module Gitlab Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) end + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) User.for_github_id(id).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_email(email) User.by_any_email(email).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord # Reads an ID from the cache. # diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 195672f5a12..047487f1d24 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -52,10 +52,12 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 07c0abcce23..b54e45de4fe 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -4,6 +4,7 @@ module Gitlab "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" end + # rubocop: disable CodeReuse/ActiveRecord def self.parse(gl_repository) match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) unless match_data @@ -16,5 +17,6 @@ module Gitlab [project, wiki] end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 5070f4e3cfe..94c15739231 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -78,6 +78,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues @@ -123,6 +124,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_labels(raw_issue) labels = [] diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 2716834f566..2bc081a6181 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -26,6 +26,7 @@ module Gitlab !!(signature_text && signed_text) end + # rubocop: disable CodeReuse/ActiveRecord def signature return unless has_signature? @@ -36,6 +37,7 @@ module Gitlab @signature = create_cached_signature! end + # rubocop: enable CodeReuse/ActiveRecord def update_signature!(cached_signature) using_keychain do |gpg_key| @@ -113,9 +115,11 @@ module Gitlab gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + # rubocop: disable CodeReuse/ActiveRecord def find_gpg_key(keyid) GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 1991911ef6a..6972bd685f7 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -5,6 +5,7 @@ module Gitlab @gpg_key = gpg_key end + # rubocop: disable CodeReuse/ActiveRecord def run GpgSignature .select(:id, :commit_sha, :project_id) @@ -12,6 +13,7 @@ module Gitlab .where(gpg_key_primary_keyid: @gpg_key.keyids) .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb new file mode 100644 index 00000000000..e3b9c59bd6e --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional performance metrics to the grape logger +module Gitlab + module GrapeLogging + module Loggers + class PerfLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + { gitaly_calls: Gitlab::GitalyClient.get_request_count } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb new file mode 100644 index 00000000000..f3146b4dfd9 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/route_logger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the details of the action +module Gitlab + module GrapeLogging + module Loggers + class RouteLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + endpoint = request.env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + + return {} unless route + + { route: route } + rescue + # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + {} + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb index abee2afe144..3c0d7e9784a 100644 --- a/lib/gitlab/graphql/connections/keyset_connection.rb +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -6,6 +6,7 @@ module Gitlab encode(node[order_field].to_s) end + # rubocop: disable CodeReuse/ActiveRecord def sliced_nodes @sliced_nodes ||= begin @@ -17,7 +18,9 @@ module Gitlab sliced end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def paged_nodes if first && last raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") @@ -29,6 +32,7 @@ module Gitlab sliced_nodes.limit(limit_value) end end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 42ded7c286f..8fbfa1a86bf 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -19,9 +19,11 @@ module Gitlab # Returns the set of descendants of a given relation, but excluding the given # relation + # rubocop: disable CodeReuse/ActiveRecord def descendants base_and_descendants.where.not(id: descendants_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord # Returns the set of ancestors of a given relation, but excluding the given # relation @@ -29,9 +31,11 @@ module Gitlab # Passing an `upto` will stop the recursion once the specified parent_id is # reached. So all ancestors *lower* than the specified ancestor will be # included. + # rubocop: disable CodeReuse/ActiveRecord def ancestors(upto: nil) base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord # Returns a relation that includes the ancestors_base set of groups # and all their ancestors (recursively). @@ -75,6 +79,7 @@ module Gitlab # Rails thinking it's selecting data the usual way. # # If nested groups are not supported, ancestors_base is returned. + # rubocop: disable CodeReuse/ActiveRecord def all_groups return ancestors_base unless Group.supports_nested_groups? @@ -84,20 +89,22 @@ module Gitlab ancestors_table = ancestors.alias_to(groups_table) descendants_table = descendants.alias_to(groups_table) - union = SQL::Union.new([model.unscoped.from(ancestors_table), - model.unscoped.from(descendants_table)]) - relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) - .from("(#{union.to_sql}) #{model.table_name}") + .from_union([ + model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table) + ]) read_only(relation) end + # rubocop: enable CodeReuse/ActiveRecord private + # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) @@ -113,7 +120,9 @@ module Gitlab cte << parent_query cte end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants_cte cte = SQL::RecursiveCTE.new(:base_and_descendants) @@ -127,6 +136,7 @@ module Gitlab cte end + # rubocop: enable CodeReuse/ActiveRecord def groups_table model.arel_table diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index d11fcc6a3e3..4edc251facb 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -22,6 +22,7 @@ module Gitlab # # @param [Object] start first project id for the range # @param [Object] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord def bulk_migrate(start, finish) projects = build_relation(start, finish) @@ -29,6 +30,7 @@ module Gitlab migrate(project) end end + # rubocop: enable CodeReuse/ActiveRecord # Flag a project to be migrated # @@ -43,6 +45,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def build_relation(start, finish) relation = Project table = Project.arel_table @@ -52,6 +55,7 @@ module Gitlab relation end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 303b05e6a9a..22edd5f999d 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -21,6 +21,7 @@ module Gitlab !range_from.nil? && range_from == range_to end + # rubocop: disable CodeReuse/ActiveRecord def self.project_id_batches(&block) Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) @@ -28,20 +29,25 @@ module Gitlab yield ids.min, ids.max end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.hashed_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord def self.relation_summary(relation_name, relation) relation_count = relation.count @@ -62,6 +68,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def self.listing(relation_name, relation) relation_count = relation_summary(relation_name, relation) return unless relation_count > 0 @@ -78,6 +85,7 @@ module Gitlab break if index + 1 >= limit end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 0eb9b77634a..2f6c4db12bb 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Cache.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f322fe831b8..63d2882c5b2 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Queues.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index 07e6f707998..f1ea1ffe1be 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::SharedState.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 5408a1a6838..0b6cc893db1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,5 +1,8 @@ module Gitlab class Highlight + TIMEOUT_BACKGROUND = 30.seconds + TIMEOUT_FOREGROUND = 3.seconds + def self.highlight(blob_name, blob_content, repository: nil, plain: false) new(blob_name, blob_content, repository: repository) .highlight(blob_content, continue: false, plain: plain) @@ -51,11 +54,20 @@ module Gitlab end def highlight_rich(text, continue: true) - @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + tag = lexer.tag + tokens = lexer.lex(text, continue: continue) + Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + rescue Timeout::Error => e + Gitlab::Sentry.track_exception(e) + highlight_plain(text) rescue highlight_plain(text) end + def timeout_time + Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + end + def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3f3f10596c5..a8b93f1d4b2 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -28,6 +28,7 @@ module Gitlab end # Tries to identify a user based on a user identifier (e.g. "user-123"). + # rubocop: disable CodeReuse/ActiveRecord def identify_using_user(identifier) user_id = identifier.gsub("user-", "") @@ -35,6 +36,7 @@ module Gitlab User.find_by(id: user_id) end end + # rubocop: enable CodeReuse/ActiveRecord # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). def identify_using_ssh_key(identifier) diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 80857061933..5b3f30d894a 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -8,6 +8,7 @@ module Gitlab # attributes - The attributes/columns to set. # relation - An ActiveRecord::Relation to use for finding the ID of the row # when using MySQL. + # rubocop: disable CodeReuse/ActiveRecord def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -20,6 +21,7 @@ module Gitlab result.first || relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb new file mode 100644 index 00000000000..8414954d141 --- /dev/null +++ b/lib/gitlab/import/logger.rb @@ -0,0 +1,9 @@ +module Gitlab + module Import + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'importer' + end + end + end +end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 8ba70700dc1..97dc1a987c4 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module MergeRequestHelpers include DatabaseHelpers + # rubocop: disable CodeReuse/ActiveRecord def create_merge_request_without_hooks(project, attributes, iid) # This work must be wrapped in a transaction as otherwise we can leave # behind incomplete data in the event of an error. This can then lead @@ -39,7 +40,9 @@ module Gitlab # existing row. [project.merge_requests.find_by(iid: iid), true] end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false) # These fields are set so we can create the correct merge request # diffs. @@ -65,6 +68,7 @@ module Gitlab diff.save diff.save_git_content end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index 83134bb0769..7cbf653dd97 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -53,7 +53,7 @@ module Gitlab end def self.lock_file_path(project) - return unless project.export_path || object_storage? + return unless project.export_path || export_file_exists? lock_path = project.import_export_shared.archive_path @@ -83,8 +83,8 @@ module Gitlab errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end - def object_storage? - project.export_project_object_exists? + def export_file_exists? + project.export_file_exists? end end end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..4f29bdcea2c 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..17796430811 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].find { |f| File.file?(f) } end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f69f98a78a3..2bed470514b 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -19,6 +19,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - :issue_assignees - snippets: - :award_emoji @@ -45,6 +48,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - pipelines: - notes: - :author @@ -64,6 +70,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :prometheus_metrics - :project_badges - :ci_cd_settings @@ -108,6 +115,9 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + prometheus_metrics: + - :common + - :identifier snippets: - :expired_at merge_request_diff: @@ -133,6 +143,16 @@ excluded_attributes: - :event_id project_badges: - :group_id + resource_label_events: + - :reference + - :reference_html + - :epic_id + hooks: + - :token + - :encrypted_token + - :encrypted_token_iv + - :encrypted_url + - :encrypted_url_iv methods: labels: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..3d693d23c99 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -136,9 +136,18 @@ module Gitlab return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten + null_iid_pipelines = [] # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift + while relation_item = (tree_array.shift || null_iid_pipelines.shift) + if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any? + # Move pipelines with NULL IIDs to the end + # so they don't clash with existing IIDs. + null_iid_pipelines << relation_item + + next + end + # The transaction at this level is less speedy than one single transaction # But we can't have it in the upper level or GC won't get rid of the AR objects # after we save the batch. @@ -199,7 +208,11 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) + end + + def nil_iid_pipeline?(relation_key, relation_item) + relation_key == 'pipelines' && relation_item['iid'].nil? end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 81807ed659c..2486b1e4921 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -86,7 +86,6 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when 'Ci::Pipeline' then setup_pipeline end update_user_references @@ -94,6 +93,8 @@ module Gitlab update_group_references remove_duplicate_assignees + setup_pipeline if @relation_name == 'Ci::Pipeline' + reset_tokens! remove_encrypted_attributes! end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..59a74083395 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_upload else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -51,7 +49,7 @@ module Gitlab @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end - def save_on_object_storage + def save_upload upload = ImportExportUpload.find_or_initialize_by(project: @project) File.open(archive_file) { |file| upload.export_file = file } @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 505810964bc..b5657a36998 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -1,7 +1,7 @@ module Gitlab # Class for counting and caching the number of issuables per state. class IssuablesCountForState - # The name of the RequestStore cache key. + # The name of the Gitlab::SafeRequestStore cache key. CACHE_KEY = :issuables_count_for_state # The state values that can be safely casted to a Symbol. @@ -10,12 +10,7 @@ module Gitlab # finder - The finder class to use for retrieving the issuables. def initialize(finder) @finder = finder - @cache = - if RequestStore.active? - RequestStore[CACHE_KEY] ||= initialize_cache - else - initialize_cache - end + @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end def for_state_or_opened(state = nil) diff --git a/lib/gitlab/kubernetes/cluster_role_binding.rb b/lib/gitlab/kubernetes/cluster_role_binding.rb new file mode 100644 index 00000000000..ebea8aff5be --- /dev/null +++ b/lib/gitlab/kubernetes/cluster_role_binding.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ClusterRoleBinding + attr_reader :name, :cluster_role_name, :subjects + + def initialize(name, cluster_role_name, subjects) + @name = name + @cluster_role_name = cluster_role_name + @subjects = subjects + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.roleRef = role_ref + resource.subjects = subjects + end + end + + private + + def metadata + { name: name } + end + + def role_ref + { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: cluster_role_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 530ccf88053..4a1bdf34c3e 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,6 +3,9 @@ module Gitlab module Helm HELM_VERSION = '2.7.2'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze + SERVICE_ACCOUNT = 'tiller'.freeze + CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze + CLUSTER_ROLE = 'cluster-admin'.freeze end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index d65374cc23b..e21bc531444 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -9,7 +9,17 @@ module Gitlab def install(command) namespace.ensure_exists! + + create_service_account(command) + create_cluster_role_binding(command) create_config_map(command) + + kubeclient.create_pod(command.pod_resource) + end + + def update(command) + namespace.ensure_exists! + update_config_map(command) kubeclient.create_pod(command.pod_resource) end @@ -32,6 +42,12 @@ module Gitlab kubeclient.delete_pod(pod_name, namespace.name) end + def get_config_map(config_map_name) + namespace.ensure_exists! + + kubeclient.get_config_map(config_map_name, namespace.name) + end + private attr_reader :kubeclient, :namespace @@ -41,6 +57,56 @@ module Gitlab kubeclient.create_config_map(config_map_resource) end end + + def update_config_map(command) + command.config_map_resource.tap do |config_map_resource| + kubeclient.update_config_map(config_map_resource) + end + end + + def create_service_account(command) + command.service_account_resource.tap do |service_account_resource| + break unless service_account_resource + + if service_account_exists?(service_account_resource) + kubeclient.update_service_account(service_account_resource) + else + kubeclient.create_service_account(service_account_resource) + end + end + end + + def create_cluster_role_binding(command) + command.cluster_role_binding_resource.tap do |cluster_role_binding_resource| + break unless cluster_role_binding_resource + + if cluster_role_binding_exists?(cluster_role_binding_resource) + kubeclient.update_cluster_role_binding(cluster_role_binding_resource) + else + kubeclient.create_cluster_role_binding(cluster_role_binding_resource) + end + end + end + + def service_account_exists?(resource) + resource_exists? do + kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) + end + end + + def cluster_role_binding_exists?(resource) + resource_exists? do + kubeclient.get_cluster_role_binding(resource.metadata.name) + end + end + + def resource_exists? + yield + rescue ::Kubeclient::HttpError => e + raise e unless e.error_code == 404 + + false + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index afcfd109de0..6752f2cff43 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -3,7 +3,9 @@ module Gitlab module Helm module BaseCommand def pod_resource - Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate end def generate_script @@ -26,6 +28,14 @@ module Gitlab Gitlab::Kubernetes::ConfigMap.new(name, files).generate end + def service_account_resource + nil + end + + def cluster_role_binding_resource + nil + end + def file_names files.keys end @@ -34,6 +44,10 @@ module Gitlab raise "Not implemented" end + def rbac? + raise "Not implemented" + end + def files raise "Not implemented" end @@ -47,6 +61,10 @@ module Gitlab def namespace Gitlab::Kubernetes::Helm::NAMESPACE end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end end end end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index a4546509515..c7046a9ea75 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -6,9 +6,10 @@ module Gitlab attr_reader :name, :files - def initialize(name:, files:) + def initialize(name:, files:, rbac:) @name = name @files = files + @rbac = rbac end def generate_script @@ -17,15 +18,62 @@ module Gitlab ].join("\n") end + def rbac? + @rbac + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + private def init_helm_command - tls_flags = "--tiller-tls" \ - " --tiller-tls-verify --tls-ca-cert #{files_dir}/ca.pem" \ - " --tiller-tls-cert #{files_dir}/cert.pem" \ - " --tiller-tls-key #{files_dir}/key.pem" + command = %w[helm init] + init_command_flags + + command.shelljoin + " >/dev/null\n" + end + + def init_command_flags + tls_flags + optional_service_account_flag + end + + def tls_flags + [ + '--tiller-tls', + '--tiller-tls-verify', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tiller-tls-cert', "#{files_dir}/cert.pem", + '--tiller-tls-key', "#{files_dir}/key.pem" + ] + end + + def optional_service_account_flag + return [] unless rbac? + + ['--service-account', service_account_name] + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end - "helm init #{tls_flags} >/dev/null" + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 9672f80687e..1be7924d6ac 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -6,10 +6,11 @@ module Gitlab attr_reader :name, :files, :chart, :version, :repository - def initialize(name:, chart:, files:, version: nil, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil) @name = name @chart = chart @version = version + @rbac = rbac @files = files @repository = repository end @@ -22,6 +23,10 @@ module Gitlab ].compact.join("\n") end + def rbac? + @rbac + end + private def init_command @@ -29,28 +34,51 @@ module Gitlab end def repository_command - "helm repo add #{name} #{repository}" if repository + ['helm', 'repo', 'add', name, repository].shelljoin if repository end def script_command - init_flags = "--name #{name}#{optional_tls_flags}#{optional_version_flag}" \ - " --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE}" \ - " -f /data/helm/#{name}/config/values.yaml" + command = ['helm', 'install', chart] + install_command_flags + + command.shelljoin + " >/dev/null\n" + end + + def install_command_flags + name_flag = ['--name', name] + namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + value_flag = ['-f', "/data/helm/#{name}/config/values.yaml"] - "helm install #{chart} #{init_flags} >/dev/null\n" + name_flag + + optional_tls_flags + + optional_version_flag + + optional_rbac_create_flag + + namespace_flag + + value_flag + end + + def optional_rbac_create_flag + return [] unless rbac? + + # jupyterhub helm chart is using rbac.enabled + # https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/master/jupyterhub + %w[--set rbac.create=true,rbac.enabled=true] end def optional_version_flag - " --version #{version}" if version + return [] unless version + + ['--version', version] end def optional_tls_flags - return unless files.key?(:'ca.pem') + return [] unless files.key?(:'ca.pem') - " --tls" \ - " --tls-ca-cert #{files_dir}/ca.pem" \ - " --tls-cert #{files_dir}/cert.pem" \ - " --tls-key #{files_dir}/key.pem" + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] end end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 6e5d3388405..95192b11c0d 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -2,9 +2,10 @@ module Gitlab module Kubernetes module Helm class Pod - def initialize(command, namespace_name) + def initialize(command, namespace_name, service_account_name: nil) @command = command @namespace_name = namespace_name + @service_account_name = service_account_name end def generate @@ -12,13 +13,14 @@ module Gitlab spec[:volumes] = volumes_specification spec[:containers][0][:volumeMounts] = volume_mounts_specification + spec[:serviceAccountName] = service_account_name if service_account_name ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end private - attr_reader :command, :namespace_name, :kubeclient, :config_map + attr_reader :command, :namespace_name, :service_account_name def container_specification { diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb new file mode 100644 index 00000000000..74188046739 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class UpgradeCommand + include BaseCommand + + attr_reader :name, :chart, :version, :repository, :files + + def initialize(name, chart:, files:, rbac:, version: nil, repository: nil) + @name = name + @chart = chart + @rbac = rbac + @version = version + @files = files + @repository = repository + end + + def generate_script + super + [ + init_command, + repository_command, + script_command + ].compact.join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "upgrade-#{name}" + end + + private + + def init_command + 'helm init --client-only >/dev/null' + end + + def repository_command + "helm repo add #{name} #{repository}" if repository + end + + def script_command + upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \ + " --reset-values" \ + " --install" \ + " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ + " -f /data/helm/#{name}/config/values.yaml" + + "helm upgrade #{name} #{chart}#{upgrade_flags} >/dev/null\n" + end + + def optional_version_flag + " --version #{version}" if version + end + + def optional_tls_flags + return unless files.key?(:'ca.pem') + + " --tls" \ + " --tls-ca-cert #{files_dir}/ca.pem" \ + " --tls-cert #{files_dir}/cert.pem" \ + " --tls-key #{files_dir}/key.pem" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb new file mode 100644 index 00000000000..588238de608 --- /dev/null +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require 'uri' + +module Gitlab + module Kubernetes + # Wrapper around Kubeclient::Client to dispatch + # the right message to the client that can respond to the message. + # We must have a kubeclient for each ApiGroup as there is no + # other way to use the Kubeclient gem. + # + # See https://github.com/abonas/kubeclient/issues/348. + class KubeClient + include Gitlab::Utils::StrongMemoize + + SUPPORTED_API_GROUPS = [ + 'api', + 'apis/rbac.authorization.k8s.io', + 'apis/extensions' + ].freeze + + # Core API methods delegates to the core api group client + delegate :get_pods, + :get_secrets, + :get_config_map, + :get_namespace, + :get_pod, + :get_secret, + :get_service, + :get_service_account, + :delete_pod, + :create_config_map, + :create_namespace, + :create_pod, + :create_secret, + :create_service_account, + :update_config_map, + :update_service_account, + to: :core_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client + delegate :create_cluster_role_binding, + :get_cluster_role_binding, + :update_cluster_role_binding, + to: :rbac_client + + # Deployments resource is currently on the apis/extensions api group + delegate :get_deployments, + to: :extensions_client + + # non-entity methods that can only work with the core client + # as it uses the pods/log resource + delegate :get_pod_log, + :watch_pod_log, + to: :core_client + + def initialize(api_prefix, api_groups = ['api'], api_version = 'v1', **kubeclient_options) + raise ArgumentError unless check_api_groups_supported?(api_groups) + + @api_prefix = api_prefix + @api_groups = api_groups + @api_version = api_version + @kubeclient_options = kubeclient_options + end + + def discover! + clients.each(&:discover) + end + + def clients + hashed_clients.values + end + + def core_client + hashed_clients['api'] + end + + def rbac_client + hashed_clients['apis/rbac.authorization.k8s.io'] + end + + def extensions_client + hashed_clients['apis/extensions'] + end + + def hashed_clients + strong_memoize(:hashed_clients) do + @api_groups.map do |api_group| + api_url = join_api_url(@api_prefix, api_group) + [api_group, ::Kubeclient::Client.new(api_url, @api_version, **@kubeclient_options)] + end.to_h + end + end + + private + + def check_api_groups_supported?(api_groups) + api_groups.all? {|api_group| SUPPORTED_API_GROUPS.include?(api_group) } + end + + def join_api_url(api_prefix, api_path) + url = URI.parse(api_prefix) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") + + url.to_s + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account.rb b/lib/gitlab/kubernetes/service_account.rb new file mode 100644 index 00000000000..d58fc1c3976 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccount + attr_reader :name, :namespace_name + + def initialize(name, namespace_name) + @name = name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata) + end + + private + + def metadata + { + name: name, + namespace: namespace_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account_token.rb b/lib/gitlab/kubernetes/service_account_token.rb new file mode 100644 index 00000000000..2e912b26c09 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account_token.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccountToken + attr_reader :name, :service_account_name, :namespace_name + + def initialize(name, service_account_name, namespace_name) + @name = name + @service_account_name = service_account_name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata, type: service_acount_token_type) + end + + private + + # as per https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#to-create-additional-api-tokens + def service_acount_token_type + 'kubernetes.io/service-account-token' + end + + def metadata + { + name: name, + namespace: namespace_name, + annotations: { + "kubernetes.io/service-account.name": service_account_name + } + } + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 2f07fde406c..11d1300e51e 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -10,6 +10,7 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new end + # rubocop: disable CodeReuse/ActiveRecord def create! association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend @@ -17,6 +18,7 @@ module Gitlab record.attributes = attributes end end + # rubocop: enable CodeReuse/ActiveRecord def url raw_data.url || '' diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b04d678cf98..c5bde681365 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -113,6 +113,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| @@ -133,6 +134,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| @@ -193,6 +195,7 @@ module Gitlab issuable.update_attribute(:label_ids, label_ids) end + # rubocop: disable CodeReuse/ActiveRecord def import_comments(issuable_type) resource_type = "#{issuable_type}_comments".to_sym @@ -213,7 +216,9 @@ module Gitlab create_comments(comments) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| @@ -238,6 +243,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def discard_inserted_comments(comments, last_note) last_note_attrs = nil diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index de55382d3ad..7db4a54267e 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -55,12 +55,14 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def milestone if raw_data.milestone.present? milestone = MilestoneFormatter.new(project, raw_data.milestone) project.milestones.find_by(milestone.find_condition) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index c3eed12e739..e9663650903 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -13,6 +13,7 @@ module Gitlab :labels end + # rubocop: disable CodeReuse/ActiveRecord def create! params = attributes.except(:project) service = ::Labels::FindOrCreateService.new(nil, project, params) @@ -22,6 +23,7 @@ module Gitlab label end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 6d8055622f1..3794380e2d0 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -29,6 +29,7 @@ module Gitlab .try(:id) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_external_uid return nil unless id @@ -40,6 +41,7 @@ module Gitlab .first .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index e58927a40b9..3d7c049c17f 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -30,7 +30,7 @@ module Gitlab end def self.build - RequestStore[self.cache_key] ||= new(self.full_log_path) + Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path) end def self.full_log_path diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index c205f348023..04107296ae3 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -6,9 +6,15 @@ module Gitlab include Gitlab::Metrics::Methods attach_to :active_record + IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze + def sql(event) return unless current_transaction + payload = event.payload + + return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 3d588918adf..10cb1e7127e 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -83,7 +83,7 @@ module Gitlab def open_file(params, key) allowed_paths = [ - FileUploader.root, + ::FileUploader.root, Gitlab.config.uploads.storage_path, File.join(Rails.root, 'public/uploads/tmp') ] diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index fd5de73c526..fab7a43dd30 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -53,6 +53,7 @@ module Gitlab @first_collection_page_count = first_collection_page.total_pages end + # rubocop: disable CodeReuse/ActiveRecord def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) @@ -60,5 +61,6 @@ module Gitlab .except(:select) .size end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb new file mode 100644 index 00000000000..8db331dcb9f --- /dev/null +++ b/lib/gitlab/null_request_store.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Used by Gitlab::SafeRequestStore +module Gitlab + # The methods `begin!`, `clear!`, and `end!` are not defined because they + # should only be called directly on `RequestStore`. + class NullRequestStore + def store + {} + end + + def active? + end + + def read(key) + end + + def [](key) + end + + def write(key, value) + value + end + + def []=(key, value) + value + end + + def exist?(key) + false + end + + def fetch(key, &block) + yield + end + + def delete(key, &block) + yield(key) if block_given? + end + end +end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 22332474945..ca5d49eedb9 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -26,6 +26,7 @@ module Gitlab @filename = filename end + # rubocop: disable CodeReuse/ActiveRecord def rotate!(old_key:, new_key:) old_key ||= Gitlab::Application.secrets.otp_key_base @@ -47,7 +48,9 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def rollback! ActiveRecord::Base.transaction do CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| @@ -55,6 +58,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb new file mode 100644 index 00000000000..a9f6cfb19cb --- /dev/null +++ b/lib/gitlab/patch/prependable.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# We're patching `ActiveSupport::Concern` in +# config/initializers/0_as_concern.rb +# +# We want to patch `ActiveSupport::Concern` for two reasons: +# 1. Allow defining class methods via: `class_methods` method +# 2. Allow `prepended do; end` work like `included do; end` +# If we don't need anything above, we don't need this patch nor the concern! + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Patch + module Prependable + class MultiplePrependedBlocks < StandardError + def initialize + super "Cannot define multiple 'prepended' blocks for a Concern" + end + end + + def prepend_features(base) + return false if prepended?(base) + + super + + if const_defined?(:ClassMethods) + klass_methods = const_get(:ClassMethods) + base.singleton_class.prepend klass_methods + base.instance_variable_set(:@_prepended_class_methods, klass_methods) + end + + if instance_variable_defined?(:@_prepended_block) + base.class_eval(&@_prepended_block) + end + + true + end + + def class_methods + super + + if instance_variable_defined?(:@_prepended_class_methods) + const_get(:ClassMethods).prepend @_prepended_class_methods + end + end + + def prepended(base = nil, &block) + if base.nil? + raise MultiplePrependedBlocks if + instance_variable_defined?(:@_prepended_block) + + @_prepended_block = block + else + super + end + end + + def prepended?(base) + index = base.ancestors.index(base) + + base.ancestors[0...index].index(self) + end + end + end +end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 92a308a12dc..fda61000f6a 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -15,6 +15,7 @@ module Gitlab Gitlab::CurrentSettings.performance_bar_allowed_group_id end + # rubocop: disable CodeReuse/ActiveRecord def self.allowed_user_ids Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) @@ -26,6 +27,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def self.expire_allowed_user_ids_cache Rails.cache.delete(ALLOWED_USER_IDS_KEY) diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index f2825db59ae..37ff32b1296 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -23,7 +23,7 @@ module Gitlab end subscribe('sql.active_record') do |_, start, finish, _, data| - if RequestStore.active? && RequestStore.store[:peek_enabled] + if Gitlab::SafeRequestStore.store[:peek_enabled] # data[:cached] is only available starting from Rails 5.1.0 # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 # Before that, data[:name] was set to 'CACHE' diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index c5bb4648572..15391b1330b 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -34,6 +34,7 @@ module Gitlab # # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. + # rubocop: disable CodeReuse/ActiveRecord def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get @@ -76,6 +77,7 @@ module Gitlab result end + # rubocop: enable CodeReuse/ActiveRecord def self.create_custom_logger(logger, private_token: nil) return unless logger @@ -135,6 +137,7 @@ module Gitlab result end + # rubocop: disable CodeReuse/ActiveRecord def self.log_load_times_by_model(logger) return unless logger.respond_to?(:load_times_by_model) @@ -146,6 +149,7 @@ module Gitlab logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end + # rubocop: enable CodeReuse/ActiveRecord def self.print_by_total_time(result, options = {}) default_options = { sort_method: :total_time } diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index e3da1634fa5..448c3f3a7d8 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -49,13 +49,11 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped .with .recursive(cte.to_arel) - .select_from_union(union) + .select_from_union(relations) end private diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index 7d0c00c7f36..ed2287dcc7e 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -24,11 +24,9 @@ module Gitlab user.groups.joins(:shared_projects).select_for_project_authorization ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped - .select_from_union(union) + .select_from_union(relations) end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 62f9e538c04..71e9cc7f238 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,6 +29,7 @@ module Gitlab @blobs_count ||= blobs.count end + # rubocop: disable CodeReuse/ActiveRecord def limited_notes_count return @limited_notes_count if defined?(@limited_notes_count) @@ -42,6 +43,7 @@ module Gitlab @limited_notes_count end + # rubocop: enable CodeReuse/ActiveRecord def wiki_blobs_count @wiki_blobs_count ||= wiki_blobs.count @@ -118,9 +120,11 @@ module Gitlab @notes ||= notes_finder(nil) end + # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def commits @commits ||= find_commits(query) diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/project_service_logger.rb new file mode 100644 index 00000000000..e84dca97962 --- /dev/null +++ b/lib/gitlab/project_service_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class ProjectServiceLogger < Gitlab::JsonLogger + def self.file_name_noext + 'integrations_json' + end + end +end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index bb1172f82a1..a240d090074 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -5,7 +5,7 @@ module Gitlab MUTEX = Mutex.new extend self - def load_groups_from_yaml(file_name = 'additional_metrics.yml') + def load_groups_from_yaml(file_name) yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index e91c6fb2e27..d696a8fc00c 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -4,10 +4,13 @@ module Gitlab include ActiveModel::Model attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true def self.common_metrics - AdditionalMetricsParser.load_groups_from_yaml + ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric)) + end end # EE only diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 8534afcc849..fa86d2dfd6c 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,6 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( @@ -17,6 +18,7 @@ module Gitlab ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index e3af217b202..09f8f1103d2 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -4,6 +4,7 @@ module Gitlab class AdditionalMetricsEnvironmentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( @@ -13,6 +14,7 @@ module Gitlab ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index c2626581897..3a609a795ba 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,6 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +26,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index b62910c8de6..4d8b136d7af 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -2,6 +2,7 @@ module Gitlab module Prometheus module Queries class EnvironmentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| environment_slug = environment.slug @@ -19,6 +20,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index b1bf3ca4143..a03ce07b6a1 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -29,5 +29,21 @@ module Gitlab def read(key) backend.read(cache_key(key)) end + + def write(key, value) + backend.write(cache_key(key), value) + end + + def fetch_without_caching_false(key, &block) + value = read(key) + return value if value + + value = yield + + # Don't cache false values + write(key, value) if value + + value + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 2ec871f0754..d95024fccf7 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -1,23 +1,80 @@ module Gitlab module RepositoryCacheAdapter extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize class_methods do - # Wraps around the given method and caches its output in Redis and an instance - # variable. + # Caches and strongly memoizes the method. # # This only works for methods that do not take any arguments. - def cache_method(name, fallback: nil, memoize_only: false) - original = :"_uncached_#{name}" + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end - alias_method(original, name) + # Caches truthy values from the method. All values are strongly memoized, + # and cached in RequestStore. + # + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be cached. + def cache_method_asymmetrically(name) + uncached_name = alias_uncached_method(name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do - __send__(original) # rubocop:disable GitlabSecurity/PublicSend + cache_method_output_asymmetrically(name) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend end end end + + # Strongly memoizes the method. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value + # is not memoized. + def memoize_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + memoize_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Prepends "_uncached_" to the target method name + # + # Returns the uncached method name + def alias_uncached_method(name) + uncached_name = :"_uncached_#{name}" + + alias_method(uncached_name, name) + + uncached_name + end + end + + # RequestStore-backed RepositoryCache to be used. Should be overridden by + # the including class + def request_store_cache + raise NotImplementedError end # RepositoryCache to be used. Should be overridden by the including class @@ -30,65 +87,93 @@ module Gitlab raise NotImplementedError end - # Caches the supplied block both in a cache and in an instance variable. + # Caches and strongly memoizes the supplied block. # - # The cache key and instance variable are named the same way as the value of - # the `key` argument. + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + cache.fetch(name, &block) + end + end + + # Caches truthy values from the supplied block. All values are strongly + # memoized, and cached in RequestStore. # - # This method will return `nil` if the corresponding instance variable is also - # set to `nil`. This ensures we don't keep yielding the block when it returns - # `nil`. + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. # - # key - The name of the key to cache the data in. - # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, memoize_only: false, &block) - ivar = cache_instance_variable_name(key) - - if instance_variable_defined?(ivar) - instance_variable_get(ivar) - else - # If the repository doesn't exist and a fallback was specified we return - # that value inmediately. This saves us Rugged/gRPC invocations. - return fallback unless fallback.nil? || cache.repository.exists? - - begin - value = - if memoize_only - yield - else - cache.fetch(key, &block) - end - - instance_variable_set(ivar, value) - rescue Gitlab::Git::Repository::NoRepository - # Even if the above `#exists?` check passes these errors might still - # occur (for example because of a non-existing HEAD). We want to - # gracefully handle this and not cache anything - fallback + # name - The name of the method to be cached. + def cache_method_output_asymmetrically(name, &block) + memoize_method_output(name) do + request_store_cache.fetch(name) do + cache.fetch_without_caching_false(name, &block) end end end + # Strongly memoizes the supplied block. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value is + # not memoized. + def memoize_method_output(name, fallback: nil, &block) + no_repository_fallback(name, fallback: fallback) do + strong_memoize(memoizable_name(name), &block) + end + end + + # Returns the fallback value if the repository does not exist + def no_repository_fallback(name, fallback: nil, &block) + # Avoid unnecessary gRPC invocations + return fallback if fallback && fallback_early?(name) + + yield + rescue Gitlab::Git::Repository::NoRepository + # Even if the `#exists?` check in `fallback_early?` passes, these errors + # might still occur (for example because of a non-existing HEAD). We + # want to gracefully handle this and not memoize anything. + fallback + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) - methods.each do |key| - unless cached_methods.include?(key.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + methods.each do |name| + unless cached_methods.include?(name.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" next end - cache.expire(key) + cache.expire(name) - ivar = cache_instance_variable_name(key) - - remove_instance_variable(ivar) if instance_variable_defined?(ivar) + clear_memoization(memoizable_name(name)) end + + expire_request_store_method_caches(methods) end private - def cache_instance_variable_name(key) - :"@#{key.to_s.tr('?!', '')}" + def memoizable_name(name) + "#{name.to_s.tr('?!', '')}" + end + + def expire_request_store_method_caches(methods) + methods.each do |name| + request_store_cache.expire(name) + end + end + + # All cached repository methods depend on the existence of a Git repository, + # so if the repository doesn't exist, we already know not to call it. + def fallback_early?(method_name) + # Avoid infinite loop + return false if method_name == :exists? + + !exists? end end end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index fef536ecb0b..8562d4515aa 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -2,7 +2,7 @@ module Gitlab class RequestContext class << self def client_ip - RequestStore[:client_ip] + Gitlab::SafeRequestStore[:client_ip] end end @@ -13,7 +13,7 @@ module Gitlab def call(env) req = Rack::Request.new(env) - RequestStore[:client_ip] = req.ip + Gitlab::SafeRequestStore[:client_ip] = req.ip @app.call(env) end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb new file mode 100644 index 00000000000..4e82353adb6 --- /dev/null +++ b/lib/gitlab/safe_request_store.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SafeRequestStore + NULL_STORE = Gitlab::NullRequestStore.new + + class << self + # These methods should always run directly against RequestStore + delegate :clear!, :begin!, :end!, :active?, to: :RequestStore + + # These methods will run against NullRequestStore if RequestStore is disabled + delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store + end + + def self.store + if RequestStore.active? + RequestStore + else + NULL_STORE + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 1e45d074e0a..d1cf8e10228 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -62,10 +62,13 @@ module Gitlab without_count ? collection.without_count : collection end + # rubocop: disable CodeReuse/ActiveRecord def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_issues_count return @limited_issues_count if @limited_issues_count @@ -77,14 +80,19 @@ module Gitlab sum = issues(public_only: true).limit(count_limit).count @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_merge_requests_count @limited_merge_requests_count ||= merge_requests.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_milestones_count @limited_milestones_count ||= milestones.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord def single_commit_result? false @@ -100,6 +108,7 @@ module Gitlab limit_projects.search(query) end + # rubocop: disable CodeReuse/ActiveRecord def issues(finder_params = {}) issues = IssuesFinder.new(current_user, finder_params).execute unless default_project_filter @@ -115,13 +124,17 @@ module Gitlab issues.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) milestones.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def merge_requests merge_requests = MergeRequestsFinder.new(current_user).execute unless default_project_filter @@ -137,13 +150,16 @@ module Gitlab merge_requests.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'projects' end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a17cd27e82d..89d2028d7b0 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -225,6 +225,7 @@ module Gitlab # Ex. # remove_keys_not_found_in_db # + # rubocop: disable CodeReuse/ActiveRecord def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? @@ -243,6 +244,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord # Iterate over all ssh key IDs from gitlab shell, in batches # @@ -326,9 +328,11 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # rubocop: disable CodeReuse/ActiveRecord def exists?(storage, dir_name) Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) end + # rubocop: enable CodeReuse/ActiveRecord protected @@ -368,15 +372,6 @@ module Gitlab private - def gitlab_projects(shard_name, disk_path) - Gitlab::Git::GitlabProjects.new( - shard_name, - disk_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb deleted file mode 100644 index 5512afa45a8..00000000000 --- a/lib/gitlab/sidekiq_throttler.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - class SidekiqThrottler - class << self - def execute! - if Gitlab::CurrentSettings.sidekiq_throttling_enabled? - require 'sidekiq-limit_fetch' - - Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| - Sidekiq::Queue[queue].limit = queue_limit - end - end - end - - private - - def queue_limit - @queue_limit ||= - begin - factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor - (factor * Sidekiq.options[:concurrency]).ceil - end - end - end - end -end diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index 466554e398c..0c76378d51c 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -40,9 +40,11 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_by_iid(iid) collection.find_by(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 93e00ab75a1..b308fd9637f 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -36,6 +36,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_action(from, to) environment = project.environments.find_by(name: from) return unless environment @@ -50,6 +51,7 @@ module Gitlab actions.first end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb index acba84b54b4..ee78f0f832e 100644 --- a/lib/gitlab/slash_commands/issue_search.rb +++ b/lib/gitlab/slash_commands/issue_search.rb @@ -9,6 +9,7 @@ module Gitlab "issue search <your query>" end + # rubocop: disable CodeReuse/ActiveRecord def execute(match) issues = collection.search(match[:query]).limit(QUERY_LIMIT) @@ -18,6 +19,7 @@ module Gitlab Presenters::Access.new(issues).not_found end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 4f86b3e8f73..95e37dfbdab 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -30,13 +30,17 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def snippet_titles limit_snippets.search(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def snippet_blobs limit_snippets.search_code(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'snippet_blobs' diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index b19aa6dea35..1c87c56c45e 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,5 +1,6 @@ module Gitlab class StringRegexMarker < StringRangeMarker + # rubocop: disable CodeReuse/ActiveRecord def mark(regex, group: 0, &block) ranges = [] @@ -11,5 +12,6 @@ module Gitlab super(ranges, &block) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7393574ac13..4456217017f 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,21 +1,32 @@ module Gitlab module Template class BaseTemplate - def initialize(path, project = nil) + attr_reader :category + + def initialize(path, project = nil, category: nil) @path = path + @category = category @finder = self.class.finder(project) end def name File.basename(@path, self.class.extension) end + alias_method :key, :name def content @finder.read(@path) end + # Present for compatibility with license templates, which can replace text + # like `[fullname]` with a user-specified string. This is a no-op for + # other templates + def resolve!(_placeholders = {}) + self + end + def to_json - { name: name, content: content } + { key: key, name: name, content: content } end def <=>(other) @@ -62,7 +73,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) }.sort + files.map { |f| new(f, project, category: category) }.sort end def category_directory(category) diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 831da45191f..b08d9a99e99 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads file present on Gitlab installation directory +# Searches and reads file present on GitLab installation directory module Gitlab module Template module Finders diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 29bc2393ff9..9140ace879f 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads files present on each Gitlab project repository +# Searches and reads files present on each GitLab project repository module Gitlab module Template module Finders diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fd040148a1e..deae53cc61b 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -20,7 +20,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/gitlab-ci-yml') + Rails.root.join('lib/gitlab/ci/templates') end def finder(project = nil) diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb index 880e55f71df..8d7dacc6c54 100644 --- a/lib/gitlab/temporarily_allow.rb +++ b/lib/gitlab/temporarily_allow.rb @@ -10,7 +10,7 @@ module Gitlab end def temporarily_allowed?(key) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] > 0 else TEMPORARILY_ALLOW_MUTEX.synchronize do @@ -26,11 +26,11 @@ module Gitlab end def temporarily_allow_request_store - RequestStore[:temporarily_allow] ||= Hash.new(0) + Gitlab::SafeRequestStore[:temporarily_allow] ||= Hash.new(0) end def temporarily_allow_add(key, value) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] += value else TEMPORARILY_ALLOW_MUTEX.synchronize do diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb index e387667480d..c251e78f5c5 100644 --- a/lib/gitlab/testing/request_inspector_middleware.rb +++ b/lib/gitlab/testing/request_inspector_middleware.rb @@ -35,11 +35,15 @@ module Gitlab request_headers = env_http_headers(env) status, headers, body = @app.call(env) + full_body = '' + body.each { |b| full_body << b } + request = OpenStruct.new( url: url, status_code: status, request_headers: request_headers, - response_headers: headers + response_headers: headers, + body: full_body ) log_request request diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb new file mode 100644 index 00000000000..c2955cd374c --- /dev/null +++ b/lib/gitlab/tree_summary.rb @@ -0,0 +1,119 @@ +module Gitlab + class TreeSummary + include ::Gitlab::Utils::StrongMemoize + + attr_reader :commit, :project, :path, :offset, :limit + + attr_reader :resolved_commits + private :resolved_commits + + def initialize(commit, project, params = {}) + @commit = commit + @project = project + + @path = params.fetch(:path, nil).presence + @offset = params.fetch(:offset, 0).to_i + @limit = (params.fetch(:limit, 25) || 25).to_i + + # Ensure that if multiple tree entries share the same last commit, they share + # a ::Commit instance. This prevents us from rendering the same commit title + # multiple times + @resolved_commits = {} + end + + # Creates a summary of the tree entries for a commit, within the window of + # entries defined by the offset and limit parameters. This consists of two + # return values: + # + # - An Array of Hashes containing the following keys: + # - file_name: The full path of the tree entry + # - type: One of :blob, :tree, or :submodule + # - commit: The last ::Commit to touch this entry in the tree + # - commit_path: URI of the commit in the web interface + # - An Array of the unique ::Commit objects in the first value + def summarize + summary = contents + .map { |content| build_entry(content) } + .tap { |summary| fill_last_commits!(summary) } + + [summary, commits] + end + + # Does the tree contain more entries after the given offset + limit? + def more? + all_contents[next_offset].present? + end + + # The offset of the next batch of tree entries. If more? returns false, this + # batch will be empty + def next_offset + [all_contents.size + 1, offset + limit].min + end + + private + + def contents + all_contents[offset, limit] + end + + def commits + resolved_commits.values + end + + def repository + project.repository + end + + def entry_path(entry) + File.join(*[path, entry[:file_name]].compact) + end + + def build_entry(entry) + { file_name: entry.name, type: entry.type } + end + + def fill_last_commits!(entries) + # Ensure the path is in "path/" format + ensured_path = + if path + File.join(*[path, ""]) + end + + commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) + + entries.each do |entry| + path_key = entry_path(entry) + commit = cache_commit(commits_hsh[path_key]) + + if commit + entry[:commit] = commit + entry[:commit_path] = commit_path(commit) + end + end + end + + def cache_commit(commit) + return nil unless commit.present? + + resolved_commits[commit.id] ||= commit + end + + def commit_path(commit) + Gitlab::Routing.url_helpers.project_commit_path(project, commit) + end + + def all_contents + strong_memoize(:all_contents) do + [ + *tree.trees, + *tree.blobs, + *tree.submodules + ] + end + end + + def tree + strong_memoize(:tree) { repository.tree(commit.id, path) } + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 308a95d2f09..29672d68cad 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -3,7 +3,7 @@ module Gitlab ALLOWED_SCHEMES = %w[http https ssh git].freeze def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) + regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 7797bd5fab2..5097c3253c9 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -10,6 +10,7 @@ module Gitlab .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) + .merge(usage_counters) end def to_json(force_refresh: false) @@ -22,7 +23,7 @@ module Gitlab hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, installation_type: Gitlab::INSTALLATION_TYPE, - active_user_count: User.active.count, + active_user_count: count(User.active), recorded_at: Time.now, edition: 'CE' } @@ -31,57 +32,59 @@ module Gitlab end # rubocop:disable Metrics/AbcSize + # rubocop: disable CodeReuse/ActiveRecord def system_usage_data { counts: { - assignee_lists: List.assignee.count, - boards: Board.count, - ci_builds: ::Ci::Build.count, - ci_internal_pipelines: ::Ci::Pipeline.internal.count, - ci_external_pipelines: ::Ci::Pipeline.external.count, - ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, - ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, - ci_runners: ::Ci::Runner.count, - ci_triggers: ::Ci::Trigger.count, - ci_pipeline_schedules: ::Ci::PipelineSchedule.count, - auto_devops_enabled: ::ProjectAutoDevops.enabled.count, - auto_devops_disabled: ::ProjectAutoDevops.disabled.count, - deploy_keys: DeployKey.count, - deployments: Deployment.count, - environments: ::Environment.count, - clusters: ::Clusters::Cluster.count, - clusters_enabled: ::Clusters::Cluster.enabled.count, - clusters_disabled: ::Clusters::Cluster.disabled.count, - clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, - clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, - clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, - clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, - clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, - clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, - in_review_folder: ::Environment.in_review_folder.count, - groups: Group.count, - issues: Issue.count, - keys: Key.count, - label_lists: List.label.count, - labels: Label.count, - lfs_objects: LfsObject.count, - merge_requests: MergeRequest.count, - milestone_lists: List.milestone.count, - milestones: Milestone.count, - notes: Note.count, - pages_domains: PagesDomain.count, - projects: Project.count, - projects_imported_from_github: Project.where(import_type: 'github').count, - protected_branches: ProtectedBranch.count, - releases: Release.count, - remote_mirrors: RemoteMirror.count, - snippets: Snippet.count, - todos: Todo.count, - uploads: Upload.count, - web_hooks: WebHook.count + assignee_lists: count(List.assignee), + boards: count(Board), + ci_builds: count(::Ci::Build), + ci_internal_pipelines: count(::Ci::Pipeline.internal), + ci_external_pipelines: count(::Ci::Pipeline.external), + ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), + ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), + ci_runners: count(::Ci::Runner), + ci_triggers: count(::Ci::Trigger), + ci_pipeline_schedules: count(::Ci::PipelineSchedule), + auto_devops_enabled: count(::ProjectAutoDevops.enabled), + auto_devops_disabled: count(::ProjectAutoDevops.disabled), + deploy_keys: count(DeployKey), + deployments: count(Deployment), + environments: count(::Environment), + clusters: count(::Clusters::Cluster), + clusters_enabled: count(::Clusters::Cluster.enabled), + clusters_disabled: count(::Clusters::Cluster.disabled), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), + clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), + clusters_applications_helm: count(::Clusters::Applications::Helm.installed), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), + clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + in_review_folder: count(::Environment.in_review_folder), + groups: count(Group), + issues: count(Issue), + keys: count(Key), + label_lists: count(List.label), + labels: count(Label), + lfs_objects: count(LfsObject), + merge_requests: count(MergeRequest), + milestone_lists: count(List.milestone), + milestones: count(Milestone), + notes: count(Note), + pages_domains: count(PagesDomain), + projects: count(Project), + projects_imported_from_github: count(Project.where(import_type: 'github')), + protected_branches: count(ProtectedBranch), + releases: count(Release), + remote_mirrors: count(RemoteMirror), + snippets: count(Snippet), + todos: count(Todo), + uploads: count(Upload), + web_hooks: count(WebHook) }.merge(services_usage) } end + # rubocop: enable CodeReuse/ActiveRecord def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json @@ -104,6 +107,12 @@ module Gitlab } end + def usage_counters + { + web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count + } + end + def components_usage_data { gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, @@ -112,6 +121,7 @@ module Gitlab } end + # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { JiraService: :projects_jira_active, @@ -120,9 +130,16 @@ module Gitlab PrometheusService: :projects_prometheus_active } - results = Service.unscoped.where(type: types.keys, active: true).group(:type).count - results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) + types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + end + + def count(relation, fallback: -1) + relation.count + rescue ActiveRecord::StatementInvalid + fallback end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb new file mode 100644 index 00000000000..bd0d24e4369 --- /dev/null +++ b/lib/gitlab/user_extractor.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# This class extracts all users found in a piece of text by the username or the +# email adress + +module Gitlab + class UserExtractor + # Not using `Devise.email_regexp` to filter out any chars that an email + # does not end with and not pinning the email to a start of end of a string. + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + USERNAME_REGEXP = User.reference_pattern + + def initialize(text) + @text = text + end + + # rubocop: disable CodeReuse/ActiveRecord + def users + return User.none unless @text.present? + + @users ||= User.from_union(union_relations) + end + # rubocop: enable CodeReuse/ActiveRecord + + def usernames + matches[:usernames] + end + + def emails + matches[:emails] + end + + def references + @references ||= matches.values.flatten + end + + def matches + @matches ||= { + emails: @text.scan(EMAIL_REGEXP).flatten.uniq, + usernames: @text.scan(USERNAME_REGEXP).flatten.uniq + } + end + + private + + def union_relations + relations = [] + + relations << User.by_any_email(emails) if emails.any? + relations << User.by_username(usernames) if usernames.any? + + relations + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 7b2a62fed48..d00921e6cdc 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -89,15 +89,19 @@ module Gitlab def included(base = nil) super - queue_verification(base) + queue_verification(base) if base end - alias_method :prepended, :included + def prepended(base = nil) + super + + queue_verification(base) if base + end - def extended(mod) + def extended(mod = nil) super - queue_verification(mod.singleton_class) + queue_verification(mod.singleton_class) if mod end def queue_verification(base) diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 73fc43cb590..201fcc7de7f 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -11,9 +11,11 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def all_relation Upload.all.preload(:model) end + # rubocop: enable CodeReuse/ActiveRecord def local?(upload) upload.local? diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb new file mode 100644 index 00000000000..1cd9b5295b9 --- /dev/null +++ b/lib/gitlab/web_ide_commits_counter.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module WebIdeCommitsCounter + WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze + + class << self + def increment + Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) } + end + + def total_count + Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i } + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a9629a92a50..30a8c3578d8 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,18 +22,27 @@ module Gitlab project = repository.project - { + attrs = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitConfigOptions: [], GitalyServer: { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) } } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end + + attrs end def send_git_blob(repository, blob) |