summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/branches.rb5
-rw-r--r--lib/api/entities.rb6
-rw-r--r--lib/api/merge_requests.rb3
-rw-r--r--lib/api/projects.rb23
-rw-r--r--lib/backup/manager.rb2
-rw-r--r--lib/banzai/reference_parser/issue_parser.rb3
-rw-r--r--lib/gitlab/auth.rb24
-rw-r--r--lib/gitlab/ci/build/policy.rb15
-rw-r--r--lib/gitlab/ci/build/policy/kubernetes.rb19
-rw-r--r--lib/gitlab/ci/build/policy/refs.rb43
-rw-r--r--lib/gitlab/ci/build/policy/specification.rb25
-rw-r--r--lib/gitlab/ci/yaml_processor.rb120
-rw-r--r--lib/gitlab/closing_issue_extractor.rb2
-rw-r--r--lib/gitlab/data_builder/push.rb7
-rw-r--r--lib/gitlab/diff/file.rb4
-rw-r--r--lib/gitlab/diff/file_collection/base.rb5
-rw-r--r--lib/gitlab/ee_compat_check.rb7
-rw-r--r--lib/gitlab/gfm/reference_rewriter.rb6
-rw-r--r--lib/gitlab/git.rb9
-rw-r--r--lib/gitlab/git/blob.rb2
-rw-r--r--lib/gitlab/git/commit.rb4
-rw-r--r--lib/gitlab/git/diff.rb4
-rw-r--r--lib/gitlab/git/hook.rb7
-rw-r--r--lib/gitlab/git/operation_service.rb8
-rw-r--r--lib/gitlab/git/popen.rb8
-rw-r--r--lib/gitlab/git/repository.rb124
-rw-r--r--lib/gitlab/git/rev_list.rb4
-rw-r--r--lib/gitlab/git/storage.rb1
-rw-r--r--lib/gitlab/git/storage/circuit_breaker.rb30
-rw-r--r--lib/gitlab/git/storage/health.rb2
-rw-r--r--lib/gitlab/git/storage/null_circuit_breaker.rb47
-rw-r--r--lib/gitlab/gitaly_client.rb147
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb59
-rw-r--r--lib/gitlab/gon_helper.rb1
-rw-r--r--lib/gitlab/health_checks/fs_shards_check.rb6
-rw-r--r--lib/gitlab/i18n.rb3
-rw-r--r--lib/gitlab/usage_data.rb46
-rw-r--r--lib/system_check/incoming_email/imap_authentication_check.rb45
-rw-r--r--lib/tasks/gitlab/dev.rake5
39 files changed, 685 insertions, 196 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 642c1140fcc..643c8e6fb8e 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -21,7 +21,10 @@ module API
get ':id/repository/branches' do
branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
- present paginate(branches), with: Entities::RepoBranch, project: user_project
+ # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+ Gitlab::GitalyClient.allow_n_plus_1_calls do
+ present paginate(branches), with: Entities::RepoBranch, project: user_project
+ end
end
resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 52c49e5caa9..71253f72533 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -244,7 +244,10 @@ module API
end
expose :merged do |repo_branch, options|
- options[:project].repository.merged_to_root_ref?(repo_branch.name)
+ # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
+ Gitlab::GitalyClient.allow_n_plus_1_calls do
+ options[:project].repository.merged_to_root_ref?(repo_branch.name)
+ end
end
expose :protected do |repo_branch, options|
@@ -332,6 +335,7 @@ module API
end
class IssueBasic < ProjectEntity
+ expose :closed_at
expose :labels do |issue, options|
# Avoids an N+1 query since labels are preloaded
issue.labels.map(&:title).sort
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 56d72d511da..8aa1e0216ee 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -2,7 +2,7 @@ module API
class MergeRequests < Grape::API
include PaginationParams
- before { authenticate! }
+ before { authenticate_non_get! }
helpers ::Gitlab::IssuableMetadata
@@ -55,6 +55,7 @@ module API
desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`'
end
get do
+ authenticate! unless params[:scope] == 'all'
merge_requests = find_merge_requests
options = { with: Entities::MergeRequestBasic,
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 7dc19788462..aab7a6c3f93 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -70,8 +70,11 @@ module API
optional :import_url, type: String, desc: 'URL from which the project is imported'
end
- def present_projects(options = {})
- projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+ def load_projects
+ ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+ end
+
+ def present_projects(projects, options = {})
projects = reorder_projects(projects)
projects = projects.with_statistics if params[:statistics]
projects = projects.with_issues_enabled if params[:with_issues_enabled]
@@ -111,7 +114,7 @@ module API
params[:user] = user
- present_projects
+ present_projects load_projects
end
end
@@ -124,7 +127,7 @@ module API
use :statistics_params
end
get do
- present_projects
+ present_projects load_projects
end
desc 'Create new project' do
@@ -229,6 +232,18 @@ module API
end
end
+ desc 'List forks of this project' do
+ success Entities::Project
+ end
+ params do
+ use :collection_params
+ end
+ get ':id/forks' do
+ forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute
+
+ present_projects forks
+ end
+
desc 'Update an existing project' do
success Entities::Project
end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index b9a573d3542..3cf3939994a 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -79,7 +79,7 @@ module Backup
# - 1495527122_gitlab_backup.tar
# - 1495527068_2017_05_23_gitlab_backup.tar
# - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
- next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/
+ next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
timestamp = $1.to_i
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index a65bbe23958..e0a8ca653cb 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -34,7 +34,8 @@ module Banzai
{ namespace: :owner },
{ group: [:owners, :group_members] },
:invited_groups,
- :project_members
+ :project_members,
+ :project_feature
]
}
),
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 11ace83c15c..87aeb76b66a 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,7 +2,7 @@ module Gitlab
module Auth
MissingPersonalTokenError = Class.new(StandardError)
- REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze
+ REGISTRY_SCOPES = [:read_registry].freeze
# Scopes used for GitLab API access
API_SCOPES = [:api, :read_user].freeze
@@ -13,11 +13,6 @@ module Gitlab
# Default scopes for OAuth applications that don't define their own
DEFAULT_SCOPES = [:api].freeze
- AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze
-
- # Other available scopes
- OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze
-
class << self
include Gitlab::CurrentSettings
@@ -132,7 +127,7 @@ module Gitlab
token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
- if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
+ if token && valid_scoped_token?(token, available_scopes)
Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
end
end
@@ -230,6 +225,21 @@ module Gitlab
def read_user_scope_authentication_abilities
[]
end
+
+ def available_scopes
+ API_SCOPES + registry_scopes
+ end
+
+ # Other available scopes
+ def optional_scopes
+ available_scopes + OPENID_SCOPES - DEFAULT_SCOPES
+ end
+
+ def registry_scopes
+ return [] unless Gitlab.config.registry.enabled
+
+ REGISTRY_SCOPES
+ end
end
end
end
diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb
new file mode 100644
index 00000000000..d10cc7802d4
--- /dev/null
+++ b/lib/gitlab/ci/build/policy.rb
@@ -0,0 +1,15 @@
+module Gitlab
+ module Ci
+ module Build
+ module Policy
+ def self.fabricate(specs)
+ specifications = specs.to_h.map do |spec, value|
+ self.const_get(spec.to_s.camelize).new(value)
+ end
+
+ specifications.compact
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb
new file mode 100644
index 00000000000..b20d374288f
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/kubernetes.rb
@@ -0,0 +1,19 @@
+module Gitlab
+ module Ci
+ module Build
+ module Policy
+ class Kubernetes < Policy::Specification
+ def initialize(spec)
+ unless spec.to_sym == :active
+ raise UnknownPolicyError
+ end
+ end
+
+ def satisfied_by?(pipeline)
+ pipeline.has_kubernetes_active?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb
new file mode 100644
index 00000000000..eadc0948d2f
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/refs.rb
@@ -0,0 +1,43 @@
+module Gitlab
+ module Ci
+ module Build
+ module Policy
+ class Refs < Policy::Specification
+ def initialize(refs)
+ @patterns = Array(refs)
+ end
+
+ def satisfied_by?(pipeline)
+ @patterns.any? do |pattern|
+ pattern, path = pattern.split('@', 2)
+
+ matches_path?(path, pipeline) &&
+ matches_pattern?(pattern, pipeline)
+ end
+ end
+
+ private
+
+ def matches_path?(path, pipeline)
+ return true unless path
+
+ pipeline.project_full_path == path
+ end
+
+ def matches_pattern?(pattern, pipeline)
+ return true if pipeline.tag? && pattern == 'tags'
+ return true if pipeline.branch? && pattern == 'branches'
+ return true if pipeline.source == pattern
+ return true if pipeline.source&.pluralize == pattern
+
+ if pattern.first == "/" && pattern.last == "/"
+ Regexp.new(pattern[1...-1]) =~ pipeline.ref
+ else
+ pattern == pipeline.ref
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb
new file mode 100644
index 00000000000..c317291f29d
--- /dev/null
+++ b/lib/gitlab/ci/build/policy/specification.rb
@@ -0,0 +1,25 @@
+module Gitlab
+ module Ci
+ module Build
+ module Policy
+ ##
+ # Abstract class that defines an interface of job policy
+ # specification.
+ #
+ # Used for job's only/except policy configuration.
+ #
+ class Specification
+ UnknownPolicyError = Class.new(StandardError)
+
+ def initialize(spec)
+ @spec = spec
+ end
+
+ def satisfied_by?(pipeline)
+ raise NotImplementedError
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 7582964b24e..0bd78b03448 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -5,12 +5,11 @@ module Gitlab
include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
- attr_reader :path, :cache, :stages, :jobs
+ attr_reader :cache, :stages, :jobs
- def initialize(config, path = nil)
+ def initialize(config)
@ci_config = Gitlab::Ci::Config.new(config)
@config = @ci_config.to_hash
- @path = path
unless @ci_config.valid?
raise ValidationError, @ci_config.errors.first
@@ -21,28 +20,12 @@ module Gitlab
raise ValidationError, e.message
end
- def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
- jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
- build_attributes(name)
- end
- end
-
def builds
@jobs.map do |name, _|
build_attributes(name)
end
end
- def stage_seeds(pipeline)
- seeds = @stages.uniq.map do |stage|
- builds = pipeline_stage_builds(stage, pipeline)
-
- Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
- end
-
- seeds.compact
- end
-
def build_attributes(name)
job = @jobs[name.to_sym] || {}
@@ -70,6 +53,32 @@ module Gitlab
}.compact }
end
+ def pipeline_stage_builds(stage, pipeline)
+ selected_jobs = @jobs.select do |_, job|
+ next unless job[:stage] == stage
+
+ only_specs = Gitlab::Ci::Build::Policy
+ .fabricate(job.fetch(:only, {}))
+ except_specs = Gitlab::Ci::Build::Policy
+ .fabricate(job.fetch(:except, {}))
+
+ only_specs.all? { |spec| spec.satisfied_by?(pipeline) } &&
+ except_specs.none? { |spec| spec.satisfied_by?(pipeline) }
+ end
+
+ selected_jobs.map { |_, job| build_attributes(job[:name]) }
+ end
+
+ def stage_seeds(pipeline)
+ seeds = @stages.uniq.map do |stage|
+ builds = pipeline_stage_builds(stage, pipeline)
+
+ Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+ end
+
+ seeds.compact
+ end
+
def self.validation_message(content)
return 'Please provide content of .gitlab-ci.yml' if content.blank?
@@ -83,34 +92,6 @@ module Gitlab
private
- def pipeline_stage_builds(stage, pipeline)
- builds = builds_for_stage_and_ref(
- stage, pipeline.ref, pipeline.tag?, pipeline.source)
-
- builds.select do |build|
- job = @jobs[build.fetch(:name).to_sym]
- has_kubernetes = pipeline.has_kubernetes_active?
- only_kubernetes = job.dig(:only, :kubernetes)
- except_kubernetes = job.dig(:except, :kubernetes)
-
- [!only_kubernetes && !except_kubernetes,
- only_kubernetes && has_kubernetes,
- except_kubernetes && !has_kubernetes].any?
- end
- end
-
- def jobs_for_ref(ref, tag = false, source = nil)
- @jobs.select do |_, job|
- process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
- end
- end
-
- def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
- jobs_for_ref(ref, tag, source).select do |_, job|
- job[:stage] == stage
- end
- end
-
def initial_parsing
##
# Global config
@@ -203,51 +184,6 @@ module Gitlab
raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
end
end
-
- def process?(only_params, except_params, ref, tag, source)
- if only_params.present?
- return false unless matching?(only_params, ref, tag, source)
- end
-
- if except_params.present?
- return false if matching?(except_params, ref, tag, source)
- end
-
- true
- end
-
- def matching?(patterns, ref, tag, source)
- patterns.any? do |pattern|
- pattern, path = pattern.split('@', 2)
- matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
- end
- end
-
- def matches_path?(path)
- return true unless path
-
- path == self.path
- end
-
- def matches_pattern?(pattern, ref, tag, source)
- return true if tag && pattern == 'tags'
- return true if !tag && pattern == 'branches'
- return true if source_to_pattern(source) == pattern
-
- if pattern.first == "/" && pattern.last == "/"
- Regexp.new(pattern[1...-1]) =~ ref
- else
- pattern == ref
- end
- end
-
- def source_to_pattern(source)
- if %w[api external web].include?(source)
- source
- else
- source&.pluralize
- end
- end
end
end
end
diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb
index 58f86abc5c4..243c1f1394d 100644
--- a/lib/gitlab/closing_issue_extractor.rb
+++ b/lib/gitlab/closing_issue_extractor.rb
@@ -1,7 +1,7 @@
module Gitlab
class ClosingIssueExtractor
ISSUE_CLOSING_REGEX = begin
- link_pattern = URI.regexp(%w(http https))
+ link_pattern = Banzai::Filter::AutolinkFilter::LINK_PATTERN
pattern = Gitlab.config.gitlab.issue_closing_pattern
pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))")
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 4ab5b3455a5..31a46a738c3 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -64,8 +64,11 @@ module Gitlab
# For performance purposes maximum 20 latest commits
# will be passed as post receive hook data.
- commit_attrs = commits_limited.map do |commit|
- commit.hook_attrs(with_changed_files: true)
+ # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259
+ commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do
+ commits_limited.map do |commit|
+ commit.hook_attrs(with_changed_files: true)
+ end
end
type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 1dabd4ebdd0..fcac85ff892 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -5,7 +5,7 @@ module Gitlab
delegate :new_file?, :deleted_file?, :renamed_file?,
:old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
- :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false
+ :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false
# Finding a viewer for a diff file happens based only on extension and whether the
# diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer,
@@ -166,7 +166,7 @@ module Gitlab
end
def binary?
- old_blob&.binary? || new_blob&.binary?
+ has_binary_notice? || old_blob&.binary? || new_blob&.binary?
end
def text?
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index a6007ebf531..88ae65cb468 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -22,7 +22,10 @@ module Gitlab
end
def diff_files
- @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+ # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445
+ Gitlab::GitalyClient.allow_n_plus_1_calls do
+ @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
+ end
end
def diff_file_with_old_path(old_path)
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index abd401224d8..c5a8ea12245 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -284,13 +284,18 @@ module Gitlab
EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
was found in the EE repository.
+ If you're a community contributor, don't worry, someone from
+ GitLab Inc. will take care of this, and you don't have to do anything.
+ If you're willing to help, and are ok to contribute to EE as well,
+ you're welcome to help. You could follow the instructions below.
+
#{conflicting_files_msg}
We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
branch that includes changes from `#{ce_branch}` but also specific changes
than can be applied cleanly to EE/master. In some cases, the conflicts
are trivial and you can ignore the warning from this job. As always,
- use your best judgment!
+ use your best judgement!
There are different ways to create such branch:
diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb
index b984492d369..455814a9159 100644
--- a/lib/gitlab/gfm/reference_rewriter.rb
+++ b/lib/gitlab/gfm/reference_rewriter.rb
@@ -29,6 +29,8 @@ module Gitlab
# http://gitlab.com/some/link/#1234, and code `puts #1234`'
#
class ReferenceRewriter
+ RewriteError = Class.new(StandardError)
+
def initialize(text, source_project, current_user)
@text = text
@source_project = source_project
@@ -61,6 +63,10 @@ module Gitlab
cross_reference = build_cross_reference(referable, target_project)
return reference if reference == cross_reference
+ if cross_reference.nil?
+ raise RewriteError, "Unspecified reference detected for #{referable.class.name}"
+ end
+
new_text = before + cross_reference + after
substitution_valid?(new_text) ? cross_reference : reference
end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index b4b6326cfdd..c78fe63f9b5 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -57,6 +57,15 @@ module Gitlab
def version
Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first)
end
+
+ def check_namespace!(*objects)
+ expected_namespace = self.name + '::'
+ objects.each do |object|
+ unless object.class.name.start_with?(expected_namespace)
+ raise ArgumentError, "expected object in #{expected_namespace}, got #{object}"
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 8d96826f6ee..a4336facee5 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -32,6 +32,8 @@ module Gitlab
else
blob = repository.lookup(sha)
+ next unless blob.is_a?(Rugged::Blob)
+
new(
id: blob.oid,
size: blob.size,
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 1f370686186..1957c254c28 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -413,6 +413,10 @@ module Gitlab
end
end
+ def merge_commit?
+ parent_ids.size > 1
+ end
+
private
def init_from_hash(hash)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index a23c8cf0dd1..096301d300f 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -206,6 +206,10 @@ module Gitlab
Diff.binary_message(@old_path, @new_path)
end
+ def has_binary_notice?
+ @diff.start_with?('Binary')
+ end
+
private
def init_from_rugged(rugged)
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index cc35d77c6e4..208e4bbaf60 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -83,13 +83,14 @@ module Gitlab
def call_update_hook(gl_id, oldrev, newrev, ref)
Dir.chdir(repo_path) do
stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev)
- [status.success?, stderr.presence || stdout]
+ [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe]
end
end
def retrieve_error_message(stderr, stdout)
- err_message = stderr.gets
- err_message.blank? ? stdout.gets : err_message
+ err_message = stderr.read
+ err_message = err_message.blank? ? stdout.read : err_message
+ err_message.gsub(/\R/, "<br>").html_safe
end
end
end
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index dcdec818f5e..786e2e7e8dc 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -1,6 +1,8 @@
module Gitlab
module Git
class OperationService
+ include Gitlab::Git::Popen
+
WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do
alias_method :repo_created?, :repo_created
alias_method :branch_created?, :branch_created
@@ -15,9 +17,7 @@ module Gitlab
end
# Refactoring aid
- unless new_repository.is_a?(Gitlab::Git::Repository)
- raise "expected a Gitlab::Git::Repository, got #{new_repository}"
- end
+ Gitlab::Git.check_namespace!(new_repository)
@repository = new_repository
end
@@ -152,7 +152,7 @@ module Gitlab
# (and have!) accidentally reset the ref to an earlier state, clobbering
# commits. See also https://github.com/libgit2/libgit2/issues/1534.
command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
- _, status = Gitlab::Popen.popen(
+ _, status = popen(
command,
repository.path) do |stdin|
stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00")
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
index 10c15b316f5..054d45895a5 100644
--- a/lib/gitlab/git/popen.rb
+++ b/lib/gitlab/git/popen.rb
@@ -5,17 +5,21 @@ require 'open3'
module Gitlab
module Git
module Popen
- def popen(cmd, path)
+ def popen(cmd, path, vars = {})
unless cmd.is_a?(Array)
raise "System commands must be given as an array of strings"
end
- vars = { "PWD" => path }
+ path ||= Dir.pwd
+ vars['PWD'] = path
options = { chdir: path }
cmd_output = ""
cmd_status = 0
Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+ yield(stdin) if block_given?
+ stdin.close
+
cmd_output << stdout.read
cmd_output << stderr.read
cmd_status = wait_thr.value.exitstatus
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index c499ff101b5..10ba29acbd1 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -19,6 +19,7 @@ module Gitlab
InvalidRef = Class.new(StandardError)
GitError = Class.new(StandardError)
DeleteBranchError = Class.new(StandardError)
+ CreateTreeError = Class.new(StandardError)
class << self
# Unlike `new`, `create` takes the storage path, not the storage name
@@ -474,7 +475,15 @@ module Gitlab
# diff options. The +options+ hash can also include :break_rewrites to
# split larger rewrites into delete/add pairs.
def diff(from, to, options = {}, *paths)
- Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options)
+ iterator = gitaly_migrate(:diff_between) do |is_enabled|
+ if is_enabled
+ gitaly_commit_client.diff(from, to, options.merge(paths: paths))
+ else
+ diff_patches(from, to, options, *paths)
+ end
+ end
+
+ Gitlab::Git::DiffCollection.new(iterator, options)
end
# Returns a RefName for a given SHA
@@ -489,7 +498,7 @@ module Gitlab
# Not found -> ["", 0]
# Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
- Gitlab::Popen.popen(args, @path).first.split.last
+ popen(args, @path).first.split.last
end
end
end
@@ -684,6 +693,88 @@ module Gitlab
nil
end
+ def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ OperationService.new(user, self).with_branch(
+ branch_name,
+ start_branch_name: start_branch_name,
+ start_repository: start_repository
+ ) do |start_commit|
+
+ Gitlab::Git.check_namespace!(commit, start_repository)
+
+ revert_tree_id = check_revert_content(commit, start_commit.sha)
+ raise CreateTreeError unless revert_tree_id
+
+ committer = user_to_committer(user)
+
+ create_commit(message: message,
+ author: committer,
+ committer: committer,
+ tree: revert_tree_id,
+ parents: [start_commit.sha])
+ end
+ end
+
+ def check_revert_content(target_commit, source_sha)
+ args = [target_commit.sha, source_sha]
+ args << { mainline: 1 } if target_commit.merge_commit?
+
+ revert_index = rugged.revert_commit(*args)
+ return false if revert_index.conflicts?
+
+ tree_id = revert_index.write_tree(rugged)
+ return false unless diff_exists?(source_sha, tree_id)
+
+ tree_id
+ end
+
+ def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ OperationService.new(user, self).with_branch(
+ branch_name,
+ start_branch_name: start_branch_name,
+ start_repository: start_repository
+ ) do |start_commit|
+
+ Gitlab::Git.check_namespace!(commit, start_repository)
+
+ cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha)
+ raise CreateTreeError unless cherry_pick_tree_id
+
+ committer = user_to_committer(user)
+
+ create_commit(message: message,
+ author: {
+ email: commit.author_email,
+ name: commit.author_name,
+ time: commit.authored_date
+ },
+ committer: committer,
+ tree: cherry_pick_tree_id,
+ parents: [start_commit.sha])
+ end
+ end
+
+ def check_cherry_pick_content(target_commit, source_sha)
+ args = [target_commit.sha, source_sha]
+ args << 1 if target_commit.merge_commit?
+
+ cherry_pick_index = rugged.cherrypick_commit(*args)
+ return false if cherry_pick_index.conflicts?
+
+ tree_id = cherry_pick_index.write_tree(rugged)
+ return false unless diff_exists?(source_sha, tree_id)
+
+ tree_id
+ end
+
+ def diff_exists?(sha1, sha2)
+ rugged.diff(sha1, sha2).size > 0
+ end
+
+ def user_to_committer(user)
+ Gitlab::Git.committer_hash(email: user.email, name: user.name)
+ end
+
def create_commit(params = {})
params[:message].delete!("\r")
@@ -709,9 +800,7 @@ module Gitlab
end
command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
- message, status = Gitlab::Popen.popen(
- command,
- path) do |stdin|
+ message, status = popen(command, path) do |stdin|
stdin.write(instructions.join)
end
@@ -835,7 +924,7 @@ module Gitlab
end
def with_repo_branch_commit(start_repository, start_branch_name)
- raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository)
+ Gitlab::Git.check_namespace!(start_repository)
return yield nil if start_repository.empty_repo?
@@ -950,8 +1039,8 @@ module Gitlab
@gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self)
end
- def gitaly_migrate(method, &block)
- Gitlab::GitalyClient.migrate(method, &block)
+ def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
+ Gitlab::GitalyClient.migrate(method, status: status, &block)
rescue GRPC::NotFound => e
raise NoRepository.new(e)
rescue GRPC::BadStatus => e
@@ -962,14 +1051,17 @@ module Gitlab
# Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
def branches_filter(filter: nil, sort_by: nil)
- branches = rugged.branches.each(filter).map do |rugged_ref|
- begin
- target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
- Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
- rescue Rugged::ReferenceError
- # Omit invalid branch
- end
- end.compact
+ # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
+ branches = Gitlab::GitalyClient.allow_n_plus_1_calls do
+ rugged.branches.each(filter).map do |rugged_ref|
+ begin
+ target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+ Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+ rescue Rugged::ReferenceError
+ # Omit invalid branch
+ end
+ end.compact
+ end
sort_branches(branches, sort_by)
end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 2b5785a1f08..e0943d3a3eb 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -3,6 +3,8 @@
module Gitlab
module Git
class RevList
+ include Gitlab::Git::Popen
+
attr_reader :oldrev, :newrev, :path_to_repo
def initialize(path_to_repo:, newrev:, oldrev: nil)
@@ -26,7 +28,7 @@ module Gitlab
private
def execute(args)
- output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
+ output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
unless status.zero?
raise "Got a non-zero exit code while calling out `#{args.join(' ')}`."
diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb
index e28be4b8a38..08e6c29abad 100644
--- a/lib/gitlab/git/storage.rb
+++ b/lib/gitlab/git/storage.rb
@@ -11,6 +11,7 @@ module Gitlab
end
CircuitOpen = Class.new(Inaccessible)
+ Misconfiguration = Class.new(Inaccessible)
REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index 9ea9367d4b7..1eaa2d83fb6 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -28,14 +28,26 @@ module Gitlab
def self.for_storage(storage)
cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
Hash.new do |hash, storage_name|
- hash[storage_name] = new(storage_name)
+ hash[storage_name] = build(storage_name)
end
end
cached_circuitbreakers[storage]
end
- def initialize(storage, hostname = Gitlab::Environment.hostname)
+ def self.build(storage, hostname = Gitlab::Environment.hostname)
+ config = Gitlab.config.repositories.storages[storage]
+
+ if !config.present?
+ NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured"))
+ elsif !config['path'].present?
+ NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured"))
+ else
+ new(storage, hostname)
+ end
+ end
+
+ def initialize(storage, hostname)
@storage = storage
@hostname = hostname
@@ -64,6 +76,10 @@ module Gitlab
recent_failure || too_many_failures
end
+ def failure_info
+ @failure_info ||= get_failure_info
+ end
+
# Memoizing the `storage_available` call means we only do it once per
# request when the storage is available.
#
@@ -121,10 +137,12 @@ module Gitlab
end
end
- def failure_info
- @failure_info ||= get_failure_info
+ def cache_key
+ @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
end
+ private
+
def get_failure_info
last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
redis.hmget(cache_key, :last_failure, :failure_count)
@@ -134,10 +152,6 @@ module Gitlab
FailureInfo.new(last_failure, failure_count.to_i)
end
-
- def cache_key
- @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
- end
end
end
end
diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb
index 2d723147f4f..1564e94b7f7 100644
--- a/lib/gitlab/git/storage/health.rb
+++ b/lib/gitlab/git/storage/health.rb
@@ -78,7 +78,7 @@ module Gitlab
def failing_circuit_breakers
@failing_circuit_breakers ||= failing_on_hosts.map do |hostname|
- CircuitBreaker.new(storage_name, hostname)
+ CircuitBreaker.build(storage_name, hostname)
end
end
diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb
new file mode 100644
index 00000000000..297c043d054
--- /dev/null
+++ b/lib/gitlab/git/storage/null_circuit_breaker.rb
@@ -0,0 +1,47 @@
+module Gitlab
+ module Git
+ module Storage
+ class NullCircuitBreaker
+ # These will have actual values
+ attr_reader :storage,
+ :hostname
+
+ # These will always have nil values
+ attr_reader :storage_path,
+ :failure_wait_time,
+ :failure_reset_time,
+ :storage_timeout
+
+ def initialize(storage, hostname, error: nil)
+ @storage = storage
+ @hostname = hostname
+ @error = error
+ end
+
+ def perform
+ @error ? raise(@error) : yield
+ end
+
+ def circuit_broken?
+ !!@error
+ end
+
+ def failure_count_threshold
+ 1
+ end
+
+ def last_failure
+ circuit_broken? ? Time.now : nil
+ end
+
+ def failure_count
+ circuit_broken? ? 1 : 0
+ end
+
+ def failure_info
+ Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index a3dc2cd0b60..cbd9ff406de 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -10,7 +10,24 @@ module Gitlab
OPT_OUT = 3
end
+ class TooManyInvocationsError < StandardError
+ attr_reader :call_site, :invocation_count, :max_call_stack
+
+ def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
+ @call_site = call_site
+ @invocation_count = invocation_count
+ @max_call_stack = max_call_stack
+ stacks = most_invoked_stack.join('\n') if most_invoked_stack
+
+ msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
+ msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
+
+ super(msg)
+ end
+ end
+
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+ MAXIMUM_GITALY_CALLS = 30
MUTEX = Mutex.new
private_constant :MUTEX
@@ -53,6 +70,8 @@ module Gitlab
# All Gitaly RPC call sites should use GitalyClient.call. This method
# makes sure that per-request authentication headers are set.
def self.call(storage, service, rpc, request)
+ enforce_gitaly_request_limits(:call)
+
metadata = request_metadata(storage)
metadata = yield(metadata) if block_given?
stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend
@@ -107,12 +126,100 @@ module Gitlab
private_class_method :opt_into_all_features?
def self.migrate(feature, status: MigrationStatus::OPT_IN)
+ # Enforce limits at both the `migrate` and `call` sites to ensure that
+ # problems are not hidden by a feature being disabled
+ enforce_gitaly_request_limits(:migrate)
+
is_enabled = feature_enabled?(feature, status: status)
metric_name = feature.to_s
metric_name += "_gitaly" if is_enabled
Gitlab::Metrics.measure(metric_name) do
- yield is_enabled
+ # Some migrate calls wrap other migrate calls
+ allow_n_plus_1_calls do
+ yield is_enabled
+ end
+ end
+ end
+
+ # Ensures that Gitaly is not being abuse through n+1 misuse etc
+ def self.enforce_gitaly_request_limits(call_site)
+ # Only count limits in request-response environments (not sidekiq for example)
+ return unless RequestStore.active?
+
+ # This is this actual number of times this call was made. Used for information purposes only
+ actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
+
+ # Do no enforce limits in production
+ return if Rails.env.production?
+
+ # Check if this call is nested within a allow_n_plus_1_calls
+ # block and skip check if it is
+ return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
+
+ # This is the count of calls outside of a `allow_n_plus_1_calls` block
+ # It is used for enforcement but not statistics
+ permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
+
+ count_stack
+
+ return if permitted_call_count <= MAXIMUM_GITALY_CALLS
+
+ raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
+ end
+
+ def self.allow_n_plus_1_calls
+ return yield unless RequestStore.active?
+
+ begin
+ increment_call_count(:gitaly_call_count_exception_block_depth)
+ yield
+ ensure
+ decrement_call_count(:gitaly_call_count_exception_block_depth)
+ end
+ end
+
+ def self.get_call_count(key)
+ RequestStore.store[key] || 0
+ end
+ private_class_method :get_call_count
+
+ def self.increment_call_count(key)
+ RequestStore.store[key] ||= 0
+ RequestStore.store[key] += 1
+ end
+ private_class_method :increment_call_count
+
+ def self.decrement_call_count(key)
+ RequestStore.store[key] -= 1
+ end
+ private_class_method :decrement_call_count
+
+ # Returns an estimate of the number of Gitaly calls made for this
+ # request
+ def self.get_request_count
+ return 0 unless RequestStore.active?
+
+ gitaly_migrate_count = get_call_count("gitaly_migrate_actual")
+ gitaly_call_count = get_call_count("gitaly_call_actual")
+
+ # Using the maximum of migrate and call_count will provide an
+ # indicator of how many Gitaly calls will be made, even
+ # before a feature is enabled. This provides us with a single
+ # metric, but not an exact number, but this tradeoff is acceptable
+ if gitaly_migrate_count > gitaly_call_count
+ gitaly_migrate_count
+ else
+ gitaly_call_count
+ end
+ end
+
+ def self.reset_counts
+ return unless RequestStore.active?
+
+ %w[migrate call].each do |call_site|
+ RequestStore.store["gitaly_#{call_site}_actual"] = 0
+ RequestStore.store["gitaly_#{call_site}_permitted"] = 0
end
end
@@ -124,5 +231,43 @@ module Gitlab
def self.encode(s)
s.dup.force_encoding(Encoding::ASCII_8BIT)
end
+
+ # Count a stack. Used for n+1 detection
+ def self.count_stack
+ return unless RequestStore.active?
+
+ stack_string = caller.drop(1).join("\n")
+
+ RequestStore.store[:stack_counter] ||= Hash.new
+
+ count = RequestStore.store[:stack_counter][stack_string] || 0
+ RequestStore.store[:stack_counter][stack_string] = count + 1
+ end
+ private_class_method :count_stack
+
+ # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
+ def self.max_call_count
+ return 0 unless RequestStore.active?
+
+ stack_counter = RequestStore.store[:stack_counter]
+ return 0 unless stack_counter
+
+ stack_counter.values.max
+ end
+ private_class_method :max_call_count
+
+ # Returns the stacks that calls Gitaly the most times. Used for n+1 detection
+ def self.max_stacks
+ return nil unless RequestStore.active?
+
+ stack_counter = RequestStore.store[:stack_counter]
+ return nil unless stack_counter
+
+ max = max_call_count
+ return nil if max.zero?
+
+ stack_counter.select { |_, v| v == max }.keys
+ end
+ private_class_method :max_stacks
end
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 1ba1a7830a4..cf3a3554552 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -32,20 +32,38 @@ module Gitlab
GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value
end
+ def diff(from, to, options = {})
+ from_id = case from
+ when NilClass
+ EMPTY_TREE_ID
+ when Rugged::Commit
+ from.oid
+ else
+ from
+ end
+
+ to_id = case to
+ when NilClass
+ EMPTY_TREE_ID
+ when Rugged::Commit
+ to.oid
+ else
+ to
+ end
+
+ request_params = diff_between_commits_request_params(from_id, to_id, options)
+
+ call_commit_diff(request_params, options)
+ end
+
def diff_from_parent(commit, options = {})
- request_params = commit_diff_request_params(commit, options)
- request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
- request_params[:enforce_limits] = options.fetch(:limits, true)
- request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
- request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+ request_params = diff_from_parent_request_params(commit, options)
- request = Gitaly::CommitDiffRequest.new(request_params)
- response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
- GitalyClient::DiffStitcher.new(response)
+ call_commit_diff(request_params, options)
end
def commit_deltas(commit)
- request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit))
+ request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit))
response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request)
response.flat_map { |msg| msg.deltas }
@@ -214,14 +232,29 @@ module Gitlab
private
- def commit_diff_request_params(commit, options = {})
+ def call_commit_diff(request_params, options = {})
+ request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
+ request_params[:enforce_limits] = options.fetch(:limits, true)
+ request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true)
+ request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
+
+ request = Gitaly::CommitDiffRequest.new(request_params)
+ response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
+ GitalyClient::DiffStitcher.new(response)
+ end
+
+ def diff_from_parent_request_params(commit, options = {})
parent_id = commit.parent_ids.first || EMPTY_TREE_ID
+ diff_between_commits_request_params(parent_id, commit.id, options)
+ end
+
+ def diff_between_commits_request_params(from_id, to_id, options)
{
repository: @gitaly_repo,
- left_commit_id: parent_id,
- right_commit_id: commit.id,
- paths: options.fetch(:paths, [])
+ left_commit_id: from_id,
+ right_commit_id: to_id,
+ paths: options.fetch(:paths, []).map { |path| GitalyClient.encode(path) }
}
end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 9bcc579278f..3a666c2268b 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -20,6 +20,7 @@ module Gitlab
gon.gitlab_url = Gitlab.config.gitlab.url
gon.revision = Gitlab::REVISION
gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png')
+ gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg')
if current_user
gon.current_user_id = current_user.id
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index eef97f54962..afaa59b1018 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -58,11 +58,11 @@ module Gitlab
end
def repository_storages
- @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
+ storages_paths.keys
end
def storages_paths
- @storage_paths ||= Gitlab.config.repositories.storages
+ Gitlab.config.repositories.storages
end
def exec_with_timeout(cmd_args, *args, &block)
@@ -125,7 +125,7 @@ module Gitlab
end
def storage_circuitbreaker_test(storage_name)
- Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" }
+ Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" }
rescue Gitlab::Git::Storage::Inaccessible
nil
end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 5d106b5c075..bdc0f04b56b 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -17,7 +17,8 @@ module Gitlab
'it' => 'Italiano',
'uk' => 'Українська',
'ja' => '日本語',
- 'ko' => '한국어'
+ 'ko' => '한국어',
+ 'nl_NL' => 'Nederlands'
}.freeze
def available_locales
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 36708078136..6857038dba8 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -9,12 +9,28 @@ module Gitlab
def uncached_data
license_usage_data.merge(system_usage_data)
+ .merge(features_usage_data)
+ .merge(components_usage_data)
end
def to_json(force_refresh: false)
data(force_refresh: force_refresh).to_json
end
+ def license_usage_data
+ usage_data = {
+ uuid: current_application_settings.uuid,
+ hostname: Gitlab.config.gitlab.host,
+ version: Gitlab::VERSION,
+ active_user_count: User.active.count,
+ recorded_at: Time.now,
+ mattermost_enabled: Gitlab.config.mattermost.enabled,
+ edition: 'CE'
+ }
+
+ usage_data
+ end
+
def system_usage_data
{
counts: {
@@ -54,18 +70,28 @@ module Gitlab
}
end
- def license_usage_data
- usage_data = {
- uuid: current_application_settings.uuid,
- hostname: Gitlab.config.gitlab.host,
- version: Gitlab::VERSION,
- active_user_count: User.active.count,
- recorded_at: Time.now,
- mattermost_enabled: Gitlab.config.mattermost.enabled,
- edition: 'CE'
+ def features_usage_data
+ features_usage_data_ce
+ end
+
+ def features_usage_data_ce
+ {
+ signup: current_application_settings.signup_enabled?,
+ ldap: Gitlab.config.ldap.enabled,
+ gravatar: current_application_settings.gravatar_enabled?,
+ omniauth: Gitlab.config.omniauth.enabled,
+ reply_by_email: Gitlab::IncomingEmail.enabled?,
+ container_registry: Gitlab.config.registry.enabled,
+ gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled
}
+ end
- usage_data
+ def components_usage_data
+ {
+ gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION },
+ git: { version: Gitlab::Git.version },
+ database: { adapter: Gitlab::Database.adapter_name, version: Gitlab::Database.version }
+ }
end
def services_usage
diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb
index dee108d987b..e55bea86d3f 100644
--- a/lib/system_check/incoming_email/imap_authentication_check.rb
+++ b/lib/system_check/incoming_email/imap_authentication_check.rb
@@ -4,22 +4,17 @@ module SystemCheck
set_name 'IMAP server credentials are correct?'
def check?
- if mailbox_config
- begin
- imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
- imap.starttls if config[:start_tls]
- imap.login(config[:email], config[:password])
- connected = true
- rescue
- connected = false
- end
+ if config
+ try_connect_imap
+ else
+ @error = "#{mail_room_config_path} does not have mailboxes setup"
+ false
end
-
- connected
end
def show_error
try_fixing_it(
+ "An error occurred: #{@error.class}: #{@error.message}",
'Check that the information in config/gitlab.yml is correct'
)
for_more_information(
@@ -30,15 +25,31 @@ module SystemCheck
private
- def mailbox_config
- return @config if @config
+ def try_connect_imap
+ imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
+ imap.starttls if config[:start_tls]
+ imap.login(config[:email], config[:password])
+ true
+ rescue => error
+ @error = error
+ false
+ end
+
+ def config
+ @config ||= load_config
+ end
+
+ def mail_room_config_path
+ @mail_room_config_path ||=
+ Rails.root.join('config', 'mail_room.yml').to_s
+ end
- config_path = Rails.root.join('config', 'mail_room.yml').to_s
- erb = ERB.new(File.read(config_path))
- erb.filename = config_path
+ def load_config
+ erb = ERB.new(File.read(mail_room_config_path))
+ erb.filename = mail_room_config_path
config_file = YAML.load(erb.result)
- @config = config_file[:mailboxes]&.first
+ config_file.dig(:mailboxes, 0)
end
end
end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 7ccda04a35f..3eade7bf553 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -13,7 +13,10 @@ namespace :gitlab do
args
end
- if Gitlab::EeCompatCheck.new(opts || {}).check
+ if File.basename(Rails.root) == 'gitlab-ee'
+ puts "Skipping EE projects"
+ exit 0
+ elsif Gitlab::EeCompatCheck.new(opts || {}).check
exit 0
else
exit 1