diff options
Diffstat (limited to 'lib')
88 files changed, 785 insertions, 802 deletions
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b7aadc27e71..6769855b899 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -112,9 +112,9 @@ module API can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push] title = params[:title] || deploy_keys_project.deploy_key.title - result = deploy_keys_project.update_attributes(can_push: can_push, - deploy_key_attributes: { id: params[:key_id], - title: title }) + result = deploy_keys_project.update(can_push: can_push, + deploy_key_attributes: { id: params[:key_id], + title: title }) if result present deploy_keys_project, with: Entities::DeployKeysProject diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 40df1e79bc7..b256c33c631 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -135,10 +135,13 @@ module API expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes def self.preload_relation(projects_relation, options = {}) + # Preloading tags, should be done with using only `:tags`, + # as `:tags` are defined as: `has_many :tags, through: :taggings` + # N+1 is solved then by using `subject.tags.map(&:name)` + # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 projects_relation.preload(:project_feature, :route) - .preload(:import_state) - .preload(namespace: [:route, :owner], - tags: :taggings) + .preload(:import_state, :tags) + .preload(namespace: [:route, :owner]) end end @@ -212,11 +215,15 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics def self.preload_relation(projects_relation, options = {}) + # Preloading tags, should be done with using only `:tags`, + # as `:tags` are defined as: `has_many :tags, through: :taggings` + # N+1 is solved then by using `subject.tags.map(&:name)` + # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 super(projects_relation).preload(:group) .preload(project_group_links: :group, fork_network: :root_project, forked_project_link: :forked_from_project, - forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) + forked_from_project: [:route, :forks, :tags, namespace: :route]) end def self.forks_counting_projects(projects_relation) @@ -775,33 +782,28 @@ module API class Todo < Grape::Entity expose :id - expose :project, using: Entities::ProjectIdentity, if: -> (todo, _) { todo.project_id } - expose :group, using: 'API::Entities::NamespaceBasic', if: -> (todo, _) { todo.group_id } + expose :project, using: Entities::BasicProjectDetails expose :author, using: Entities::UserBasic expose :action_name expose :target_type expose :target do |todo, options| - todo_target_class(todo.target_type).represent(todo.target, options) + Entities.const_get(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| target_type = todo.target_type.underscore - target_url = "#{todo.parent.class.to_s.underscore}_#{target_type}_url" + target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? Gitlab::Routing .url_helpers - .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend + .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body expose :state expose :created_at - - def todo_target_class(target_type) - ::API::Entities.const_get(target_type) - end end class NamespaceBasic < Grape::Entity diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 5c63ec028d9..fa828f43001 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -89,9 +89,10 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' end post ':id/environments/:environment_id/stop' do - authorize! :create_deployment, user_project + authorize! :read_environment, user_project environment = user_project.environments.find(params[:environment_id]) + authorize! :stop_environment, environment environment.stop_with_action!(current_user) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index f633dd88d06..797b04df059 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -150,12 +150,13 @@ module API end params do use :with_custom_attributes + optional :with_projects, type: Boolean, default: true, desc: 'Omit project details' end get ":id" do group = find_group!(params[:id]) options = { - with: Entities::GroupDetail, + with: params[:with_projects] ? Entities::GroupDetail : Entities::Group, current_user: current_user } diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index b4bfb677d72..e2984b08eca 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -97,6 +97,8 @@ module API current_user.admin? || parent.owned_by?(current_user) end + opts[:updated_at] = opts[:created_at] if opts[:created_at] + project = parent if parent.is_a?(Project) ::Notes::CreateService.new(project, current_user, opts).execute end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 68921ae439b..4760a1c08d7 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -80,7 +80,7 @@ module API update_params = declared_params(include_missing: false) - if hook.update_attributes(update_params) + if hook.update(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index b83da00502d..8273abe48c9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -260,7 +260,8 @@ module API :snippets_enabled, :tag_list, :visibility, - :wiki_enabled + :wiki_enabled, + :avatar ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 2071c5a62c1..51242341dba 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -58,7 +58,7 @@ module API optional :access_level, type: String, values: Ci::Runner.access_levels.keys, desc: 'The access_level of the runner' optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level, :maximum_timeout end put ':id' do runner = get_runner(params.delete(:id)) diff --git a/lib/api/services.rb b/lib/api/services.rb index 794fdab8f2b..553e8dff4b9 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -787,7 +787,7 @@ module API service = user_project.find_or_initialize_service(service_slug.underscore) service_params = declared_params(include_missing: false).merge(active: true) - if service.update_attributes(service_params) + if service.update(service_params) present service, with: Entities::ProjectService else render_api_error!('400 Bad Request', 400) @@ -807,7 +807,7 @@ module API hash.merge!(key => nil) end - unless service.update_attributes(attrs.merge(active: false)) + unless service.update(attrs.merge(active: false)) render_api_error!('400 Bad Request', 400) end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 02ef89f997f..1ca7d23203b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -25,7 +25,7 @@ module API optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' - optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' diff --git a/lib/api/users.rb b/lib/api/users.rb index e8df2c5a74a..5aaaf104dff 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -186,7 +186,7 @@ module API identity = user.identities.find_by(provider: identity_attrs[:provider]) if identity - identity.update_attributes(identity_attrs) + identity.update(identity_attrs) else identity = user.identities.build(identity_attrs) identity.save diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index bc9597df894..dbb25280849 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -18,7 +18,7 @@ module Banzai PARSE_OPTIONS = [ :FOOTNOTES, # parse footnotes. :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does). - :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. + :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze # The `:GITHUB_PRE_LANG` option is not used intentionally because diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index 1dd2855063d..dda6cd38dcd 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -21,7 +21,17 @@ module DeclarativePolicy cache = opts[:cache] || {} key = Cache.policy_key(user, subject) - cache[key] ||= class_for(subject).new(user, subject, opts) + cache[key] ||= + if Gitlab.rails5? + # to avoid deadlocks in multi-threaded environment when + # autoloading is enabled, we allow concurrent loads, + # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263 + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + class_for(subject).new(user, subject, opts) + end + else + class_for(subject).new(user, subject, opts) + end end def class_for(subject) diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index 47542194497..da3fabba39b 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -119,8 +119,8 @@ module DeclarativePolicy # a PolicyDsl which is used for registering the rule with # this class. PolicyDsl will call back into Base.enable_when, # Base.prevent_when, and Base.prevent_all_when. - def rule(&b) - rule = RuleDsl.new(self).instance_eval(&b) + def rule(&block) + rule = RuleDsl.new(self).instance_eval(&block) PolicyDsl.new(self, rule) end @@ -222,8 +222,8 @@ module DeclarativePolicy # computes the given ability and prints a helpful debugging output # showing which - def debug(ability, *a) - runner(ability).debug(*a) + def debug(ability, *args) + runner(ability).debug(*args) end desc "Unknown user" @@ -274,7 +274,7 @@ module DeclarativePolicy # # NOTE we can't use ||= here because the value might be the # boolean `false` - def cache(key, &b) + def cache(key) return @cache[key] if cached?(key) @cache[key] = yield diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb index f544dffe888..ca2eb98e3e8 100644 --- a/lib/declarative_policy/delegate_dsl.rb +++ b/lib/declarative_policy/delegate_dsl.rb @@ -7,10 +7,10 @@ module DeclarativePolicy @delegate_name = delegate_name end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - @rule_dsl.delegate(@delegate_name, m) + @rule_dsl.delegate(@delegate_name, msg) end end end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb index f11b6e9f730..c96049768a1 100644 --- a/lib/declarative_policy/policy_dsl.rb +++ b/lib/declarative_policy/policy_dsl.rb @@ -15,8 +15,8 @@ module DeclarativePolicy @rule = rule end - def policy(&b) - instance_eval(&b) + def policy(&block) + instance_eval(&block) end def enable(*abilities) @@ -31,14 +31,14 @@ module DeclarativePolicy @context_class.prevent_all_when(@rule) end - def method_missing(m, *a, &b) - return super unless @context_class.respond_to?(m) + def method_missing(msg, *args, &block) + return super unless @context_class.respond_to?(msg) - @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend + @context_class.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end - def respond_to_missing?(m) - @context_class.respond_to?(m) || super + def respond_to_missing?(msg) + @context_class.respond_to?(msg) || super end end end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 5c214408dd0..c77784cb49d 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -2,7 +2,7 @@ module DeclarativePolicy # rubocop:disable Naming/FileName PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" class << self - def with_preferred_scope(scope, &b) + def with_preferred_scope(scope) Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY] yield ensure @@ -13,12 +13,12 @@ module DeclarativePolicy # rubocop:disable Naming/FileName Thread.current[PREFERRED_SCOPE_KEY] end - def user_scope(&b) - with_preferred_scope(:user, &b) + def user_scope(&block) + with_preferred_scope(:user, &block) end - def subject_scope(&b) - with_preferred_scope(:subject, &b) + def subject_scope(&block) + with_preferred_scope(:subject, &block) end def preferred_scope=(scope) diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index e309244a3b3..407398cc770 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -8,8 +8,8 @@ module DeclarativePolicy # how that affects the actual ability decision - for that, a # `Step` is used. class Base - def self.make(*a) - new(*a).simplify + def self.make(*args) + new(*args).simplify end # true or false whether this rule passes. diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb index e948b7f2de1..7254b08eda5 100644 --- a/lib/declarative_policy/rule_dsl.rb +++ b/lib/declarative_policy/rule_dsl.rb @@ -32,13 +32,13 @@ module DeclarativePolicy Rule::DelegatedCondition.new(delegate_name, condition) end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - if @context_class.delegations.key?(m) - DelegateDsl.new(self, m) + if @context_class.delegations.key?(msg) + DelegateDsl.new(self, msg) else - cond(m.to_sym) + cond(msg.to_sym) end end end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 87f14b3b0d2..fec672f4b8c 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -127,7 +127,7 @@ module DeclarativePolicy # # For each step, we yield the step object along with the computed score # for debugging purposes. - def steps_by_score(&b) + def steps_by_score flatten_steps! if @steps.size > 50 diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a9b04c183ad..e8dbde176ef 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -139,6 +139,11 @@ module ExtractsPath def lfs_blob_ids blob_ids = tree.blobs.map(&:id) + + # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze + # the current Blob in order to determine if it's a LFS object + blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables + @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(@project.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 2760211fee8..f95e423ef22 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -50,7 +50,7 @@ module Gitaly @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info - rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded + rescue GRPC::Unavailable, GRPC::DeadlineExceeded # This will show the server as being out of date Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 87e377de4d3..b170145f013 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -7,12 +7,14 @@ module Gitlab module Access AccessDeniedError = Class.new(StandardError) - NO_ACCESS = 0 - GUEST = 10 - REPORTER = 20 - DEVELOPER = 30 - MASTER = 40 - OWNER = 50 + NO_ACCESS = 0 + GUEST = 10 + REPORTER = 20 + DEVELOPER = 30 + MAINTAINER = 40 + # @deprecated + MASTER = MAINTAINER + OWNER = 50 # Branch protection settings PROTECTION_NONE = 0 @@ -32,7 +34,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Maintainer" => MASTER + "Maintainer" => MAINTAINER } end @@ -44,10 +46,10 @@ module Gitlab def sym_options { - guest: GUEST, - reporter: REPORTER, - developer: DEVELOPER, - master: MASTER + guest: GUEST, + reporter: REPORTER, + developer: DEVELOPER, + maintainer: MAINTAINER } end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index e7283b2f9e8..589e8062226 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -48,7 +48,7 @@ module Gitlab gl_user rescue ActiveRecord::RecordInvalid => e log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" - return self, e.record.errors + [self, e.record.errors] end def gl_user diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index d5cf9e0d53a..cb2bdea755c 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation -# rubocop:disable Metrics/LineLength module Gitlab module BackgroundMigration diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 5a4e5b2c471..92096e29ef1 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 1b4a9e8a194..ccd1f9b4dba 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index c2bf42f846d..da8265a3a5f 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb index 0b785e1b056..664ead1af44 100644 --- a/lib/gitlab/background_migration/delete_diff_files.rb +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -1,45 +1,80 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab module BackgroundMigration class DeleteDiffFiles - def perform(merge_request_diff_id) - merge_request_diff = MergeRequestDiff.find_by(id: merge_request_diff_id) + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' - return unless merge_request_diff - return unless should_delete_diff_files?(merge_request_diff) + belongs_to :merge_request + has_many :merge_request_diff_files + end - MergeRequestDiff.transaction do - merge_request_diff.update_column(:state, 'without_files') - - # explain (analyze, buffers) when deleting 453 diff files: - # - # Delete on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actual time=43.265..43.265 rows=0 loops=1) - # Buffers: shared hit=2043 read=259 dirtied=254 - # -> Index Scan using index_merge_request_diff_files_on_mr_diff_id_and_order on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actu - # al time=0.466..26.317 rows=453 loops=1) - # Index Cond: (merge_request_diff_id = 463448) - # Buffers: shared hit=17 read=84 - # Planning time: 0.107 ms - # Execution time: 43.287 ms - # - MergeRequestDiffFile.where(merge_request_diff_id: merge_request_diff.id).delete_all + class MergeRequestDiffFile < ActiveRecord::Base + self.table_name = 'merge_request_diff_files' + end + + DEAD_TUPLES_THRESHOLD = 50_000 + VACUUM_WAIT_TIME = 5.minutes + + def perform(ids) + @ids = ids + + # We should reschedule until deadtuples get in a desirable + # state (e.g. < 50_000). That may take more than one reschedule. + # + if should_wait_deadtuple_vacuum? + reschedule + return end + + prune_diff_files + end + + def should_wait_deadtuple_vacuum? + return false unless Gitlab::Database.postgresql? + + diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD end private - def should_delete_diff_files?(merge_request_diff) - return false if merge_request_diff.state == 'without_files' + def reschedule + BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids]) + end + + def diffs_collection + MergeRequestDiff.where(id: @ids) + end + + def diff_files_dead_tuples_count + dead_tuple = + execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\ + "WHERE relname = 'merge_request_diff_files'")[0] - merge_request = merge_request_diff.merge_request + dead_tuple&.fetch('n_dead_tup', 0).to_i + end + + def prune_diff_files + removed = 0 + updated = 0 - return false unless merge_request.state == 'merged' - return false if merge_request_diff.id == merge_request.latest_merge_request_diff_id + MergeRequestDiff.transaction do + updated = diffs_collection.update_all(state: 'without_files') + removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all + end + + log_info("Removed #{removed} merge_request_diff_files rows, "\ + "updated #{updated} merge_request_diffs rows") + end + + def execute_statement(sql) + ActiveRecord::Base.connection.execute(sql) + end - true + def log_info(message) + Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}") end end end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index a357538a885..58df74cfa9b 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb index 22b0ac71920..103bd98af14 100644 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb index d0816ae3ed5..77c1f1ffaf0 100644 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb index 94c65459a67..cba3e21cea6 100644 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 242e3143e71..268c6083d3c 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 7088aa0860a..38fecac1bfe 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 7f243073fd0..ef50fe4adb1 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index a4ef51fd0e8..5b2b2af718a 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index d9d3d2e667b..698f5e46c0c 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/ClassLength # rubocop:disable Metrics/BlockLength # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index a976cb4c243..aa4f130538c 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -19,7 +19,7 @@ module Gitlab create_fork_networks_for_missing_projects(start_id, end_id) create_fork_networks_memberships_for_root_projects(start_id, end_id) - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY BackgroundMigrationWorker.perform_in( delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] ) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb index 8a901a9bf39..d89ce358bb9 100644 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -1,7 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/ClassLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9232f20a063..a19dc9747fb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,7 +4,7 @@ module Gitlab module BackgroundMigration # This class processes a batch of rows in `untracked_files_for_uploads` by # adding each file to the `uploads` table if it does not exist. - class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + class PopulateUntrackedUploads def perform(start_id, end_id) return unless migrate? diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a2c5acbde71..4a9a62aaeb5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -4,7 +4,7 @@ module Gitlab module PopulateUntrackedUploadsDependencies # This class is responsible for producing the attributes necessary to # track an uploaded file in the `uploads` table. - class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename @@ -134,7 +134,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 522c69a0bb1..81ca2b0a9b7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -144,7 +144,7 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb new file mode 100644 index 00000000000..609cf19187c --- /dev/null +++ b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ScheduleDiffFilesDeletion + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + belongs_to :merge_request + + include EachBatch + end + + DIFF_BATCH_SIZE = 5_000 + INTERVAL = 5.minutes + MIGRATION = 'DeleteDiffFiles' + + def perform + diffs = MergeRequestDiff + .from("(#{diffs_collection.to_sql}) merge_request_diffs") + .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id') + .select(:id) + + diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index| + ids = relation.pluck(:id) + + BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids]) + end + end + + private + + def diffs_collection + MergeRequestDiff + .joins(:merge_request) + .where("merge_requests.state = 'merged'") + .where('merge_requests.latest_merge_request_diff_id IS NOT NULL') + .where("merge_request_diffs.state NOT IN ('without_files', 'empty')") + .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id') + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 35eadf6fa93..e780f8c646b 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -29,105 +29,105 @@ module Gitlab end class Converter - def on_0(s) reset() end + def on_0(_) reset() end - def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_1(_) enable(STYLE_SWITCHES[:bold]) end - def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_3(_) enable(STYLE_SWITCHES[:italic]) end - def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_4(_) enable(STYLE_SWITCHES[:underline]) end - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_8(_) enable(STYLE_SWITCHES[:conceal]) end - def on_9(s) enable(STYLE_SWITCHES[:cross]) end + def on_9(_) enable(STYLE_SWITCHES[:cross]) end - def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_21(_) disable(STYLE_SWITCHES[:bold]) end - def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(_) disable(STYLE_SWITCHES[:bold]) end - def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_23(_) disable(STYLE_SWITCHES[:italic]) end - def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_24(_) disable(STYLE_SWITCHES[:underline]) end - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_28(_) disable(STYLE_SWITCHES[:conceal]) end - def on_29(s) disable(STYLE_SWITCHES[:cross]) end + def on_29(_) disable(STYLE_SWITCHES[:cross]) end - def on_30(s) set_fg_color(0) end + def on_30(_) set_fg_color(0) end - def on_31(s) set_fg_color(1) end + def on_31(_) set_fg_color(1) end - def on_32(s) set_fg_color(2) end + def on_32(_) set_fg_color(2) end - def on_33(s) set_fg_color(3) end + def on_33(_) set_fg_color(3) end - def on_34(s) set_fg_color(4) end + def on_34(_) set_fg_color(4) end - def on_35(s) set_fg_color(5) end + def on_35(_) set_fg_color(5) end - def on_36(s) set_fg_color(6) end + def on_36(_) set_fg_color(6) end - def on_37(s) set_fg_color(7) end + def on_37(_) set_fg_color(7) end - def on_38(s) set_fg_color_256(s) end + def on_38(stack) set_fg_color_256(stack) end - def on_39(s) set_fg_color(9) end + def on_39(_) set_fg_color(9) end - def on_40(s) set_bg_color(0) end + def on_40(_) set_bg_color(0) end - def on_41(s) set_bg_color(1) end + def on_41(_) set_bg_color(1) end - def on_42(s) set_bg_color(2) end + def on_42(_) set_bg_color(2) end - def on_43(s) set_bg_color(3) end + def on_43(_) set_bg_color(3) end - def on_44(s) set_bg_color(4) end + def on_44(_) set_bg_color(4) end - def on_45(s) set_bg_color(5) end + def on_45(_) set_bg_color(5) end - def on_46(s) set_bg_color(6) end + def on_46(_) set_bg_color(6) end - def on_47(s) set_bg_color(7) end + def on_47(_) set_bg_color(7) end - def on_48(s) set_bg_color_256(s) end + def on_48(stack) set_bg_color_256(stack) end - def on_49(s) set_bg_color(9) end + def on_49(_) set_bg_color(9) end - def on_90(s) set_fg_color(0, 'l') end + def on_90(_) set_fg_color(0, 'l') end - def on_91(s) set_fg_color(1, 'l') end + def on_91(_) set_fg_color(1, 'l') end - def on_92(s) set_fg_color(2, 'l') end + def on_92(_) set_fg_color(2, 'l') end - def on_93(s) set_fg_color(3, 'l') end + def on_93(_) set_fg_color(3, 'l') end - def on_94(s) set_fg_color(4, 'l') end + def on_94(_) set_fg_color(4, 'l') end - def on_95(s) set_fg_color(5, 'l') end + def on_95(_) set_fg_color(5, 'l') end - def on_96(s) set_fg_color(6, 'l') end + def on_96(_) set_fg_color(6, 'l') end - def on_97(s) set_fg_color(7, 'l') end + def on_97(_) set_fg_color(7, 'l') end - def on_99(s) set_fg_color(9, 'l') end + def on_99(_) set_fg_color(9, 'l') end - def on_100(s) set_bg_color(0, 'l') end + def on_100(_) set_bg_color(0, 'l') end - def on_101(s) set_bg_color(1, 'l') end + def on_101(_) set_bg_color(1, 'l') end - def on_102(s) set_bg_color(2, 'l') end + def on_102(_) set_bg_color(2, 'l') end - def on_103(s) set_bg_color(3, 'l') end + def on_103(_) set_bg_color(3, 'l') end - def on_104(s) set_bg_color(4, 'l') end + def on_104(_) set_bg_color(4, 'l') end - def on_105(s) set_bg_color(5, 'l') end + def on_105(_) set_bg_color(5, 'l') end - def on_106(s) set_bg_color(6, 'l') end + def on_106(_) set_bg_color(6, 'l') end - def on_107(s) set_bg_color(7, 'l') end + def on_107(_) set_bg_color(7, 'l') end - def on_109(s) set_bg_color(9, 'l') end + def on_109(_) set_bg_color(9, 'l') end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask @@ -188,19 +188,19 @@ module Gitlab ) end - def handle_section(s) - action = s[1] - timestamp = s[2] - section = s[3] - line = s.matched()[0...-5] # strips \r\033[0K + def handle_section(scanner) + action = scanner[1] + timestamp = scanner[2] + section = scanner[3] + line = scanner.matched()[0...-5] # strips \r\033[0K @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} end - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] + def handle_sequence(scanner) + indicator = scanner[1] + commands = scanner[2].split ';' + terminator = scanner[3] # We are only interested in color and text style changes - triggered by # sequences starting with '\e[' and ending with 'm'. Any other control diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 0bbd60d8ffe..375d8bc1ff5 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -7,14 +7,15 @@ module Gitlab module Artifacts class Metadata ParserError = Class.new(StandardError) + InvalidStreamError = Class.new(StandardError) VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} - attr_reader :file, :path, :full_version + attr_reader :stream, :path, :full_version - def initialize(file, path, **opts) - @file, @path, @opts = file, path, opts + def initialize(stream, path, **opts) + @stream, @path, @opts = stream, path, opts @full_version = read_version end @@ -103,7 +104,17 @@ module Gitlab end def gzip(&block) - Zlib::GzipReader.open(@file, &block) + raise InvalidStreamError, "Invalid stream" unless @stream + + # restart gzip reading + @stream.seek(0) + + gz = Zlib::GzipReader.new(@stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish end end end diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index db47c2f6185..7cddd2c7b7e 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -47,7 +47,7 @@ module Gitlab Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end - private # rubocop:disable Lint/UselessAccessModifier + private def entry(key, entry, metadata) factory = Entry::Factory.new(entry) diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb deleted file mode 100644 index 8788af57a67..00000000000 --- a/lib/gitlab/ci/trace/http_io.rb +++ /dev/null @@ -1,197 +0,0 @@ -## -# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) -# source: https://gitlab.com/snippets/1685610 -module Gitlab - module Ci - class Trace - class HttpIO - BUFFER_SIZE = 128.kilobytes - - InvalidURLError = Class.new(StandardError) - FailedToGetChunkError = Class.new(StandardError) - - attr_reader :uri, :size - attr_reader :tell - attr_reader :chunk, :chunk_range - - alias_method :pos, :tell - - def initialize(url, size) - raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) - - @uri = URI(url) - @size = size - @tell = 0 - end - - def close - # no-op - end - - def binmode - # no-op - end - - def binmode? - true - end - - def path - nil - end - - def url - @uri.to_s - end - - def seek(pos, where = IO::SEEK_SET) - new_pos = - case where - when IO::SEEK_END - size + pos - when IO::SEEK_SET - pos - when IO::SEEK_CUR - tell + pos - else - -1 - end - - raise 'new position is outside of file' if new_pos < 0 || new_pos > size - - @tell = new_pos - end - - def eof? - tell == size - end - - def each_line - until eof? - line = readline - break if line.nil? - - yield(line) - end - end - - def read(length = nil, outbuf = "") - out = "" - - length ||= size - tell - - until length <= 0 || eof? - data = get_chunk - break if data.empty? - - chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min - chunk_data = data.byteslice(0, chunk_bytes) - - out << chunk_data - @tell += chunk_data.bytesize - length -= chunk_data.bytesize - end - - # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality - if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out - end - - out - end - - def readline - out = "" - - until eof? - data = get_chunk - new_line = data.index("\n") - - if !new_line.nil? - out << data[0..new_line] - @tell += new_line + 1 - break - else - out << data - @tell += data.bytesize - end - end - - out - end - - def write(data) - raise NotImplementedError - end - - def truncate(offset) - raise NotImplementedError - end - - def flush - raise NotImplementedError - end - - def present? - true - end - - private - - ## - # The below methods are not implemented in IO class - # - def in_range? - @chunk_range&.include?(tell) - end - - def get_chunk - unless in_range? - response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| - http.request(request) - end - - raise FailedToGetChunkError unless response.code == '200' || response.code == '206' - - @chunk = response.body.force_encoding(Encoding::BINARY) - @chunk_range = response.content_range - - ## - # Note: If provider does not return content_range, then we set it as we requested - # Provider: minio - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: AWS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: GCS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 - @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) - end - - @chunk[chunk_offset..BUFFER_SIZE] - end - - def request - Net::HTTP::Get.new(uri).tap do |request| - request.set_range(chunk_start, BUFFER_SIZE) - end - end - - def chunk_offset - tell % BUFFER_SIZE - end - - def chunk_start - (tell / BUFFER_SIZE) * BUFFER_SIZE - end - - def chunk_end - [chunk_start + BUFFER_SIZE, size].min - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index 9bb0166c9e3..c09089d6475 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -75,19 +75,19 @@ module Gitlab @beginning_of_section_regex ||= /section_/.freeze end - def find_next_marker(s) + def find_next_marker(scanner) beginning_of_section_len = 8 - maybe_marker = s.exist?(beginning_of_section_regex) + maybe_marker = scanner.exist?(beginning_of_section_regex) if maybe_marker.nil? - s.terminate + scanner.terminate else # repositioning at the beginning of the match - s.pos += maybe_marker - beginning_of_section_len + scanner.pos += maybe_marker - beginning_of_section_len if block_given? - good_marker = yield(s) + good_marker = yield(scanner) # if not a good marker: Consuming the matched beginning_of_section_regex - s.pos += beginning_of_section_len unless good_marker + scanner.pos += beginning_of_section_len unless good_marker end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 3cf35f499cd..9147ef401da 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -60,7 +60,7 @@ module Gitlab def in_memory_application_settings with_fallback_to_fake_application_settings do - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults end end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb index 65332dfd239..1f157354ea4 100644 --- a/lib/gitlab/diff/image_point.rb +++ b/lib/gitlab/diff/image_point.rb @@ -3,11 +3,11 @@ module Gitlab class ImagePoint attr_reader :width, :height, :x, :y - def initialize(width, height, x, y) + def initialize(width, height, new_x, new_y) @width = width @height = height - @x = x - @y = y + @x = new_x + @y = new_y end def to_h diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 54783a07919..99970779c67 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -93,7 +93,7 @@ module Gitlab private - def longest_common_prefix(a, b) + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max length = 0 @@ -109,7 +109,7 @@ module Gitlab length end - def longest_common_suffix(a, b) + def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName longest_common_prefix(a.reverse, b.reverse) end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 8c72d00c1f3..ee604e66154 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -138,15 +138,23 @@ module Gitlab def ee_branch_presence_check! ee_remotes.keys.each do |remote| - [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch| - _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + output, _ = step( + "Searching #{remote}", + %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*]) - if status.zero? - @ee_remote_with_branch = remote - @ee_branch_found = branch - return true - end - end + branches = + output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size) + + next if branches.empty? + + branch = branches.first + + step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + + @ee_remote_with_branch = remote + @ee_branch_found = branch + + return true end puts @@ -271,6 +279,10 @@ module Gitlab @ee_patch_full_path ||= patches_dir.join(ee_patch_name) end + def minimal_ee_branch_name + @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '') + end + def patch_name_from_branch(branch_name) branch_name.parameterize << '.patch' end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 0b8f6cfe3cb..d1fd5dfe0cb 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -65,17 +65,17 @@ module Gitlab clean(message) end rescue ArgumentError - return nil + nil end - def encode_binary(s) - return "" if s.nil? + def encode_binary(str) + return "" if str.nil? - s.dup.force_encoding(Encoding::ASCII_8BIT) + str.dup.force_encoding(Encoding::ASCII_8BIT) end - def binary_stringio(s) - StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + def binary_stringio(str) + StringIO.new(str || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index ab6838adc6d..e998548cff9 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -21,7 +21,7 @@ module Gitlab raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid - return yield + yield ensure Gitlab::ExclusiveLease.cancel(key, uuid) end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 8953bc8c148..a91de278cf3 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -191,19 +191,19 @@ module Gitlab end end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') + str end - def escape_for_markdown(s) - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") - s = s.gsub("`", "\\~") - s = s.delete("\r") - s = s.gsub("\n", " \n") - s + def escape_for_markdown(str) + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") + str = str.gsub("`", "\\~") + str = str.delete("\r") + str = str.gsub("\n", " \n") + str end def format_content(raw_content) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 96fa94d5790..71857bd2d87 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -61,17 +61,8 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) - Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a - else - blob_references.map do |sha, path| - find(repository, sha, path, limit: blob_size_limit) - end - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a end # Returns an array of Blob instances just with the metadata, that means @@ -84,16 +75,8 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) - else - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force - end + repository.wrapped_gitaly_errors do + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) end end @@ -104,72 +87,6 @@ module Gitlab def size_could_be_lfs?(size) size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE) end - - private - - # Recursive search of blob id by path - # - # Ex. - # blog/ # oid: 1a - # app/ # oid: 2a - # models/ # oid: 3a - # file.rb # oid: 4a - # - # - # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' - # - def find_entry_by_path(repository, root_id, *path_parts) - root_tree = repository.lookup(root_id) - - entry = root_tree.find do |entry| - entry[:name] == path_parts[0] - end - - return nil unless entry - - if path_parts.size > 1 - return nil unless entry[:type] == :tree - - path_parts.shift - find_entry_by_path(repository, entry[:oid], *path_parts) - else - [:blob, :commit].include?(entry[:type]) ? entry : nil - end - end - - def submodule_blob(blob_entry, path, sha) - new( - id: blob_entry[:oid], - name: blob_entry[:name], - size: 0, - data: '', - path: path, - commit_id: sha - ) - end - - def rugged_raw(repository, sha, limit:) - blob = repository.lookup(sha) - - return unless blob.is_a?(Rugged::Blob) - - new( - id: blob.oid, - size: blob.size, - data: blob.content(limit), - binary: blob.binary? - ) - end - - # Efficient lookup to determine if object size - # and type make it a possible LFS blob without loading - # blob content into memory with repository.lookup(sha) - def possible_lfs_blob?(repository, sha) - object_header = repository.rugged.read_header(sha) - - object_header[:type] == :blob && - size_could_be_lfs?(object_header[:len]) - end end def initialize(options) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 420790f45d0..fc4711751b1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -86,9 +86,6 @@ module Gitlab # Relative path of repo attr_reader :relative_path - # Rugged repo object - attr_reader :rugged - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). @@ -112,8 +109,9 @@ module Gitlab [storage, relative_path] == [other.storage, other.relative_path] end + # This method will be removed when Gitaly reaches v1.1. def path - @path ||= File.join( + File.join( Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path ) end @@ -127,8 +125,9 @@ module Gitlab raise Gitlab::Git::CommandError.new(e.message) end + # This method will be removed when Gitaly reaches v1.1. def rugged - @rugged ||= circuit_breaker.perform do + circuit_breaker.perform do Rugged::Repository.new(path, alternates: alternate_object_directories) end rescue Rugged::RepositoryError, Rugged::OSError @@ -168,24 +167,9 @@ module Gitlab # Directly find a branch with a simple name (e.g. master) # - # force_reload causes a new Rugged repository to be instantiated - # - # This is to work around a bug in libgit2 that causes in-memory refs to - # be stale/invalid when packed-refs is changed. - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 - def find_branch(name, force_reload = false) - gitaly_migrate(:find_branch) do |is_enabled| - if is_enabled - gitaly_ref_client.find_branch(name) - else - reload_rugged if force_reload - - rugged_ref = rugged.branches[name] - if rugged_ref - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - end - end + def find_branch(name) + wrapped_gitaly_errors do + gitaly_ref_client.find_branch(name) end end @@ -197,20 +181,8 @@ module Gitlab # Returns the number of valid branches def branch_count - gitaly_migrate(:branch_names) do |is_enabled| - if is_enabled - gitaly_ref_client.count_branch_names - else - rugged.branches.each(:local).count do |ref| - begin - ref.name && ref.target # ensures the branch is valid - - true - rescue Rugged::ReferenceError - false - end - end - end + wrapped_gitaly_errors do + gitaly_ref_client.count_branch_names end end @@ -233,12 +205,8 @@ module Gitlab # Returns the number of valid tags def tag_count - gitaly_migrate(:tag_names) do |is_enabled| - if is_enabled - gitaly_ref_client.count_tag_names - else - rugged.tags.count - end + wrapped_gitaly_errors do + gitaly_ref_client.count_tag_names end end @@ -261,13 +229,8 @@ module Gitlab # # Ref names must start with `refs/`. def ref_exists?(ref_name) - gitaly_migrate(:ref_exists, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_exists?(ref_name) - else - rugged_ref_exists?(ref_name) - end + wrapped_gitaly_errors do + gitaly_ref_exists?(ref_name) end end @@ -275,12 +238,8 @@ module Gitlab # # name - The name of the tag as a String. def tag_exists?(name) - gitaly_migrate(:ref_exists_tags, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_exists?("refs/tags/#{name}") - else - rugged_tag_exists?(name) - end + wrapped_gitaly_errors do + gitaly_ref_exists?("refs/tags/#{name}") end end @@ -288,12 +247,8 @@ module Gitlab # # name - The name of the branch as a String. def branch_exists?(name) - gitaly_migrate(:ref_exists_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_exists?("refs/heads/#{name}") - else - rugged_branch_exists?(name) - end + wrapped_gitaly_errors do + gitaly_ref_exists?("refs/heads/#{name}") end end @@ -311,12 +266,8 @@ module Gitlab end def delete_all_refs_except(prefixes) - gitaly_migrate(:ref_delete_refs) do |is_enabled| - if is_enabled - gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) - else - delete_refs(*all_ref_names_except(prefixes)) - end + wrapped_gitaly_errors do + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) end end @@ -625,7 +576,13 @@ module Gitlab end def update_branch(branch_name, user:, newrev:, oldrev:) - OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + gitaly_migrate(:operation_user_update_branch) do |is_enabled| + if is_enabled + gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev) + else + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + end end def rm_branch(branch_name, user:) @@ -707,33 +664,18 @@ module Gitlab Gitlab::Git.committer_hash(email: user.email, name: user.name) end - def create_commit(params = {}) - params[:message].delete!("\r") - - Rugged::Commit.create(rugged, params) - end - # Delete the specified branch from the repository def delete_branch(branch_name) - gitaly_migrate(:delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.delete_branch(branch_name) - else - rugged.branches.delete(branch_name) - end + wrapped_gitaly_errors do + gitaly_ref_client.delete_branch(branch_name) end - rescue Rugged::ReferenceError, CommandError => e + rescue CommandError => e raise DeleteBranchError, e end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_delete_refs(*ref_names) - else - git_delete_refs(*ref_names) - end + wrapped_gitaly_errors do + gitaly_delete_refs(*ref_names) end end @@ -743,12 +685,8 @@ module Gitlab # create_branch("feature") # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") - gitaly_migrate(:create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.create_branch(ref, start_point) - else - rugged_create_branch(ref, start_point) - end + wrapped_gitaly_errors do + gitaly_ref_client.create_branch(ref, start_point) end end @@ -898,12 +836,8 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| - if is_enabled - gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) - else - rugged_fetch_source_branch(source_repository, source_branch, local_ref) - end + wrapped_gitaly_errors do + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) end end @@ -1058,18 +992,13 @@ module Gitlab end def bundle_to_disk(save_path) - gitaly_migrate(:bundle_to_disk) do |is_enabled| - if is_enabled - gitaly_repository_client.create_bundle(save_path) - else - run_git!(%W(bundle create #{save_path} --all)) - end + wrapped_gitaly_errors do + gitaly_repository_client.create_bundle(save_path) end true end - # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, @@ -1081,7 +1010,6 @@ module Gitlab start_branch_name, start_repository) end end - # rubocop:enable Metrics/ParameterLists def write_config(full_path:) return unless full_path.present? @@ -1186,7 +1114,7 @@ module Gitlab end def can_be_merged?(source_sha, target_branch) - if target_sha = find_branch(target_branch, true)&.target + if target_sha = find_branch(target_branch)&.target !gitaly_conflicts_client(source_sha, target_sha).conflicts? else false @@ -1560,52 +1488,10 @@ module Gitlab # Returns true if the given ref name exists # # Ref names must start with `refs/`. - def rugged_ref_exists?(ref_name) - raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') - - rugged.references.exist?(ref_name) - rescue Rugged::ReferenceError - false - end - - # Returns true if the given ref name exists - # - # Ref names must start with `refs/`. def gitaly_ref_exists?(ref_name) gitaly_ref_client.ref_exists?(ref_name) end - # Returns true if the given tag exists - # - # name - The name of the tag as a String. - def rugged_tag_exists?(name) - !!rugged.tags[name] - end - - # Returns true if the given branch exists - # - # name - The name of the branch as a String. - def rugged_branch_exists?(name) - rugged.branches.exists?(name) - - # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. - # Whatever code calls this method shouldn't have to deal with that so - # instead we just return `false` (which is true since a branch doesn't - # exist when it has an invalid name). - rescue Rugged::ReferenceError - false - end - - def rugged_create_branch(ref, start_point) - rugged_ref = rugged.branches.create(ref, start_point) - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'} - - raise InvalidRef.new("Invalid reference #{start_point}") - end - def gitaly_copy_gitattributes(revision) gitaly_repository_client.apply_gitattributes(revision) end @@ -1698,20 +1584,6 @@ module Gitlab remote_update(remote_name, url: url) end - def git_delete_refs(*ref_names) - instructions = ref_names.map do |ref| - "delete #{ref}\x00\x00" - end - - message, status = run_git(%w[update-ref --stdin -z]) do |stdin| - stdin.write(instructions.join) - end - - unless status.zero? - raise GitError.new("Could not delete refs #{ref_names}: #{message}") - end - end - def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end @@ -1762,6 +1634,12 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def create_commit(params = {}) + params[:message].delete!("\r") + + Rugged::Commit.create(rugged, params) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 5fdad077eea..2ba68343aa5 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -12,35 +12,12 @@ module Gitlab end # This method returns an array of new commit references - def new_refs - repository.rev_list(including: newrev, excluding: :all).split("\n") - end - - # Finds newly added objects - # Returns an array of shas + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1233 # - # Can skip objects which do not have a path using required_path: true - # This skips commit objects and root trees, which might not be needed when - # looking for blobs - # - # When given a block it will yield objects as a lazy enumerator so - # the caller can limit work done instead of processing megabytes of data - def new_objects(options: [], require_path: nil, not_in: nil, &lazy_block) - opts = { - including: newrev, - options: options, - excluding: not_in.nil? ? :all : not_in, - require_path: require_path - } - - get_objects(opts, &lazy_block) - end - - def all_objects(options: [], require_path: nil, &lazy_block) - get_objects(including: :all, - options: options, - require_path: require_path, - &lazy_block) + def new_refs + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + repository.rev_list(including: newrev, excluding: :all).split("\n") + end end private diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db7c29be94b..35808149b90 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,6 +2,8 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) @@ -26,7 +28,7 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @@ -40,6 +42,8 @@ module Gitlab end def check(cmd, changes) + @changes = changes + check_protocol! check_valid_actor! check_active_user! @@ -58,7 +62,7 @@ module Gitlab when *DOWNLOAD_COMMANDS check_download_access! when *PUSH_COMMANDS - check_push_access!(changes) + check_push_access! end true @@ -218,7 +222,7 @@ module Gitlab end end - def check_push_access!(changes) + def check_push_access! if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end @@ -235,17 +239,15 @@ module Gitlab return if changes.blank? # Allow access this is needed for EE. - check_change_access!(changes) + check_change_access! end - def check_change_access!(changes) + def check_change_access! # If there are worktrees with a HEAD pointing to a non-existent object, # calls to `git rev-list --all` will fail in git 2.15+. This should also # clear stale lock files. project.repository.clean_stale_repository_files - changes_list = Gitlab::ChangesList.new(changes) - # Iterate over all changes to find if user allowed all of them to be applied changes_list.each.with_index do |change, index| first_change = index == 0 @@ -321,6 +323,10 @@ module Gitlab protected + def changes_list + @changes_list ||= Gitlab::ChangesList.new(changes) + end + def user return @user if defined?(@user) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 66e781a8e5b..58a4060cc96 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -401,8 +401,8 @@ module Gitlab path.read.chomp end - def self.timestamp(t) - Google::Protobuf::Timestamp.new(seconds: t.to_i) + def self.timestamp(time) + Google::Protobuf::Timestamp.new(seconds: time.to_i) end # The default timeout on all Gitaly calls diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 72e1e59d8df..6a97cd8ed17 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -399,8 +399,8 @@ module Gitlab end end - def encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + def encode_repeated(array) + Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } ) end def call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index b1a01b185e6..aa7e03301f5 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -25,10 +25,12 @@ module Gitlab def conflicts? list_conflict_files.any? - rescue GRPC::FailedPrecondition - # The server raises this exception when it encounters ConflictSideMissing, which - # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent - # file in one of the trees. + rescue GRPC::FailedPrecondition, GRPC::Unknown + # The server raises FailedPrecondition when it encounters + # ConflictSideMissing, which means a conflict exists but its `theirs` or + # `ours` data is nil due to a non-existent file in one of the trees. + # + # GRPC::Unknown comes from Rugged::ReferenceError and Rugged::OdbError. true end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index ab2c61f6782..555733d1834 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -68,6 +68,22 @@ module Gitlab raise Gitlab::Git::Repository::InvalidRef, ex end + def user_update_branch(branch_name, user, newrev, oldrev) + request = Gitaly::UserUpdateBranchRequest.new( + repository: @gitaly_repo, + branch_name: encode_binary(branch_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + newrev: encode_binary(newrev), + oldrev: encode_binary(oldrev) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, pre_receive_error + end + end + def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 2e1076d1f66..ad898278353 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -9,7 +9,7 @@ module Gitlab end def info - GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new) + GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout) end end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 02fcb413abd..8e530de174d 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -60,8 +60,8 @@ module Gitlab private - def method_missing(m, *args, &block) - @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(msg, *args, &block) + @hash.public_send(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 46b49128140..5070f4e3cfe 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -200,27 +200,27 @@ module Gitlab "Status: #{name}" end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') + str end - def escape_for_markdown(s) + def escape_for_markdown(str) # No headings and lists - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") # No inline code - s = s.gsub("`", "\\`") + str = str.gsub("`", "\\`") # Carriage returns make me sad - s = s.delete("\r") + str = str.delete("\r") # Markdown ignores single newlines, but we need them as <br />. - s = s.gsub("\n", " \n") + str = str.gsub("\n", " \n") - s + str end def create_label(name) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 6d2278d0876..2716834f566 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -39,7 +39,7 @@ module Gitlab def update_signature!(cached_signature) using_keychain do |gpg_key| - cached_signature.update_attributes!(attributes(gpg_key)) + cached_signature.update!(attributes(gpg_key)) end @signature = cached_signature diff --git a/lib/gitlab/http_io.rb b/lib/gitlab/http_io.rb new file mode 100644 index 00000000000..ce24817db54 --- /dev/null +++ b/lib/gitlab/http_io.rb @@ -0,0 +1,193 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = "") + out = "" + + length ||= size - tell + + until length <= 0 || eof? + data = get_chunk + break if data.empty? + + chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize + end + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 8b8e48aac76..ac827cbe1ca 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -47,7 +47,7 @@ module Gitlab def ensure_default_member! @project.project_members.destroy_all - ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 60d5fa4d29a..af9b880ef9e 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -16,7 +16,8 @@ module Gitlab ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), + ImportSource.new('manifest', 'Manifest file', nil) ].freeze class << self diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index da43bd0af4b..15c5ece2350 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -1,6 +1,10 @@ module Gitlab # Helper methods to do with Kubernetes network services & resources module Kubernetes + def self.build_header_hash + Hash.new { |h, k| h[k] = [] } + end + # This is the comand that is run to start a terminal session. Kubernetes # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` EXEC_COMMAND = URI.encode_www_form( @@ -37,13 +41,14 @@ module Gitlab selectors: { pod: pod_name, container: container["name"] }, url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], - headers: Hash.new { |h, k| h[k] = [] }, + headers: ::Gitlab::Kubernetes.build_header_hash, created_at: created_at } end end def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) + terminal[:headers] ||= ::Gitlab::Kubernetes.build_header_hash terminal[:headers]['Authorization'] << "Bearer #{token}" terminal[:max_session_time] = max_session_time terminal[:ca_pem] = ca_pem if ca_pem.present? diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 344784c866f..db04356a5e9 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -53,7 +53,7 @@ module Gitlab end def config_file - ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../config/gitlab.yml', __dir__) end end end diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb new file mode 100644 index 00000000000..4d6034fb956 --- /dev/null +++ b/lib/gitlab/manifest_import/manifest.rb @@ -0,0 +1,81 @@ +# Class to parse manifest file and build a list of repositories for import +# +# <manifest> +# <remote review="https://android-review.googlesource.com/" /> +# <project path="platform-common" name="platform" /> +# <project path="platform/art" name="platform/art" /> +# <project path="platform/device" name="platform/device" /> +# </manifest> +# +# 1. Project path must be uniq and can't be part of other project path. +# For example, you can't have projects with 'foo' and 'foo/bar' paths. +# 2. Remote must be present with review attribute so GitLab knows +# where to fetch source code +module Gitlab + module ManifestImport + class Manifest + attr_reader :parsed_xml, :errors + + def initialize(file) + @parsed_xml = Nokogiri::XML(file) { |config| config.strict } + @errors = [] + rescue Nokogiri::XML::SyntaxError + @errors = ['The uploaded file is not a valid XML file.'] + end + + def projects + raw_projects.each_with_index.map do |project, i| + { + id: i, + name: project['name'], + path: project['path'], + url: repository_url(project['name']) + } + end + end + + def valid? + return false if @errors.any? + + unless validate_remote + @errors << 'Make sure a <remote> tag is present and is valid.' + end + + unless validate_projects + @errors << 'Make sure every <project> tag has name and path attributes.' + end + + @errors.empty? + end + + private + + def validate_remote + remote.present? && URI.parse(remote).host + rescue URI::Error + false + end + + def validate_projects + raw_projects.all? do |project| + project['name'] && project['path'] + end + end + + def repository_url(name) + URI.join(remote, name).to_s + end + + def remote + return @remote if defined?(@remote) + + remote_tag = parsed_xml.css('manifest > remote').first + @remote = remote_tag['review'] if remote_tag + end + + def raw_projects + @raw_projects ||= parsed_xml.css('manifest > project') + end + end + end +end diff --git a/lib/gitlab/manifest_import/project_creator.rb b/lib/gitlab/manifest_import/project_creator.rb new file mode 100644 index 00000000000..b5967c93735 --- /dev/null +++ b/lib/gitlab/manifest_import/project_creator.rb @@ -0,0 +1,41 @@ +module Gitlab + module ManifestImport + class ProjectCreator + attr_reader :repository, :destination, :current_user + + def initialize(repository, destination, current_user) + @repository = repository + @destination = destination + @current_user = current_user + end + + def execute + group_full_path, _, project_path = repository[:path].rpartition('/') + group_full_path = File.join(destination.full_path, group_full_path) if destination + group = create_group_with_parents(group_full_path) + + params = { + import_url: repository[:url], + import_type: 'manifest', + namespace_id: group.id, + path: project_path, + name: project_path, + visibility_level: destination.visibility_level + } + + Projects::CreateService.new(current_user, params).execute + end + + private + + def create_group_with_parents(full_path) + params = { + group_path: full_path, + visibility_level: destination.visibility_level + } + + Groups::NestedCreateService.new(current_user, params).execute + end + end + end +end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 66f30e3b397..04135dac4ff 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -162,7 +162,6 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. - # rubocop:disable Gitlab/ModuleWithInstanceVariables def pool if influx_metrics_enabled? if @pool.nil? @@ -180,7 +179,6 @@ module Gitlab @pool end end - # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index b11520a79bb..f3290e3149c 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -1,5 +1,3 @@ -# rubocop:disable Style/ClassVars - module Gitlab module Metrics # Class for tracking timing information about method calls diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 9753be6d5c3..18f91db98fc 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -84,7 +84,7 @@ module Gitlab def open_file(params, key) ::UploadedFile.from_params( params, key, - Gitlab.config.uploads.storage_path) + [FileUploader.root, Gitlab.config.uploads.storage_path]) end end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index 15b8beacf60..e3da1634fa5 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -24,7 +24,7 @@ module Gitlab user.projects.select_for_project_authorization, # The personal projects of the user. - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects that belong directly to any of the groups the user has # access to. diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index ad87540e6c2..7d0c00c7f36 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -15,7 +15,7 @@ module Gitlab user.projects.select_for_project_authorization, # Personal projects - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects of groups the user is a member of user.groups_projects.select_for_project_authorization, diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index e222541992a..a17cd27e82d 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -92,21 +92,13 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end relative_path = "#{name}.git" - cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - GitalyGitlabProjects.new(storage, relative_path) - else - # The timeout ensures the subprocess won't hang forever - gitlab_projects(storage, relative_path) - end - end + cmd = GitalyGitlabProjects.new(storage, relative_path) success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success @@ -126,12 +118,8 @@ module Gitlab # fetch_remote(my_repo, "upstream") # def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - gitaly_migrate(:fetch_remote) do |is_enabled| - if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - else - local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) - end + wrapped_gitaly_errors do + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) end end @@ -389,28 +377,6 @@ module Gitlab ) end - def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - vars = { force: forced, tags: !no_tags, prune: prune } - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars[:ssh_key] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars[:known_hosts] = ssh_auth.ssh_known_hosts - end - end - - cmd = gitlab_projects(storage_name, repository_relative_path) - - success = cmd.fetch_remote(remote, git_timeout, vars) - - raise Error, cmd.output unless success - - success - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -440,10 +406,6 @@ module Gitlab Gitlab.config.gitlab_shell.git_timeout end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) } - end - def wrapped_gitaly_errors yield rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 59331c827af..de8b6ec69ce 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -58,7 +58,7 @@ module Gitlab if raw_credentials.present? url.sub!("#{raw_credentials}@", '') - user, password = raw_credentials.split(':') + user, _, password = raw_credentials.partition(':') @credentials ||= { user: user.presence, password: password.presence } end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 55c899912f9..a9629a92a50 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -98,16 +98,12 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'RawPatchRequest' => Gitaly::RawPatchRequest.new( - gitaly_diff_or_patch_hash(repository, diff_refs) - ).to_json - } - else - workhorse_diff_or_patch_hash(repository, diff_refs) - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } [ SEND_DATA_HEADER, @@ -220,14 +216,6 @@ module Gitlab } end - def workhorse_diff_or_patch_hash(repository, diff_refs) - { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } - end - def gitaly_diff_or_patch_hash(repository, diff_refs) { repository: repository.gitaly_repository, diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index be0d97370d0..e877ab10248 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -11,7 +11,7 @@ module Rouge @tag = tag end - def stream(tokens, &b) + def stream(tokens) is_first = true token_lines(tokens) do |line| yield "\n" unless is_first diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 6df7fe81437..f431352b61e 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -20,16 +20,22 @@ namespace :gettext do end task :regenerate do + pot_file = 'locale/gitlab.pot' # Remove all translated files, this speeds up finding FileUtils.rm Dir['locale/**/gitlab.*'] # remove the `pot` file to ensure it's completely regenerated - FileUtils.rm_f 'locale/gitlab.pot' + FileUtils.rm_f pot_file Rake::Task['gettext:find'].invoke # leave only the required changes. `git checkout -- locale/*/gitlab.po` + # Remove timestamps from the pot file + pot_content = File.read pot_file + pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '') + File.write pot_file, pot_content + puts <<~MSG All done. Please commit the changes to `locale/gitlab.pot`. diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 83dd870fa31..26cbf0740b6 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -1,6 +1,6 @@ namespace :gitlab do namespace :import do - desc "GitLab | Add all users to all projects (admin users are added as masters)" + desc "GitLab | Add all users to all projects (admin users are added as maintainers)" task all_users_to_all_projects: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) @@ -10,7 +10,7 @@ namespace :gitlab do ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER) + ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER) end desc "GitLab | Add a specific user to all projects (as a developer)" diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 78e18992a8e..f548a266b99 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -8,7 +8,7 @@ namespace :gitlab do @uploader_class = args.uploader_class.constantize @model_class = args.model_class.constantize - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) end def enqueue_batch(batch, index) diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 5dc85b2baea..4b9cb59eab5 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -28,7 +28,7 @@ class UploadedFile @tempfile = File.new(path, 'rb') end - def self.from_params(params, field, upload_path) + def self.from_params(params, field, upload_paths) unless params["#{field}.path"] raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"] @@ -37,7 +37,8 @@ class UploadedFile file_path = File.realpath(params["#{field}.path"]) - unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact) + paths = Array(upload_paths) << Dir.tmpdir + unless self.allowed_path?(file_path, paths.compact) raise InvalidPathError, "insecure path used '#{file_path}'" end |