summaryrefslogtreecommitdiff
path: root/lib/gitlab
diff options
context:
space:
mode:
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/git/blob.rb22
-rw-r--r--lib/gitlab/git/repository.rb16
-rw-r--r--lib/gitlab/git/wiki_page.rb3
-rw-r--r--lib/gitlab/gitaly_client.rb21
-rw-r--r--lib/gitlab/gitaly_client/blob_service.rb20
-rw-r--r--lib/gitlab/gitaly_client/health_check_service.rb19
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb17
-rw-r--r--lib/gitlab/health_checks/gitaly_check.rb53
-rw-r--r--lib/gitlab/import_export/command_line_util.rb4
-rw-r--r--lib/gitlab/import_export/repo_saver.rb2
-rw-r--r--lib/gitlab/import_export/wiki_repo_saver.rb2
-rw-r--r--lib/gitlab/project_search_results.rb2
-rw-r--r--lib/gitlab/search_results.rb66
-rw-r--r--lib/gitlab/snippet_search_results.rb2
14 files changed, 212 insertions, 37 deletions
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index f421bf69e8f..81e46028752 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -34,7 +34,7 @@ module Gitlab
def raw(repository, sha)
Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
if is_enabled
- Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+ repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
else
rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
end
@@ -70,11 +70,19 @@ module Gitlab
# Returns array of Gitlab::Git::Blob
# Does not guarantee blob data will be set
def batch_lfs_pointers(repository, blob_ids)
- blob_ids.lazy
- .select { |sha| possible_lfs_blob?(repository, sha) }
- .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
- .select(&:lfs_pointer?)
- .force
+ return [] if blob_ids.empty?
+
+ repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled|
+ if is_enabled
+ repository.gitaly_blob_client.batch_lfs_pointers(blob_ids)
+ else
+ blob_ids.lazy
+ .select { |sha| possible_lfs_blob?(repository, sha) }
+ .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
+ .select(&:lfs_pointer?)
+ .force
+ end
+ end
end
def binary?(data)
@@ -258,7 +266,7 @@ module Gitlab
Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
@data = begin
if is_enabled
- Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data
+ repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data
else
repository.lookup(id).content
end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index d666362de0d..d7c712e75c5 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1268,6 +1268,18 @@ module Gitlab
success || gitlab_projects_error
end
+ def bundle_to_disk(save_path)
+ gitaly_migrate(:bundle_to_disk) do |is_enabled|
+ if is_enabled
+ gitaly_repository_client.create_bundle(save_path)
+ else
+ run_git!(%W(bundle create #{save_path} --all))
+ end
+ end
+
+ true
+ end
+
# rubocop:disable Metrics/ParameterLists
def multi_action(
user, branch_name:, message:, actions:,
@@ -1319,6 +1331,10 @@ module Gitlab
@gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self)
end
+ def gitaly_blob_client
+ @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self)
+ end
+
def gitaly_conflicts_client(our_commit_oid, their_commit_oid)
Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid)
end
diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb
index a06bac4414f..669ae11a423 100644
--- a/lib/gitlab/git/wiki_page.rb
+++ b/lib/gitlab/git/wiki_page.rb
@@ -1,7 +1,7 @@
module Gitlab
module Git
class WikiPage
- attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical
+ attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data
# This class is meant to be serializable so that it can be constructed
# by Gitaly and sent over the network to GitLab.
@@ -21,6 +21,7 @@ module Gitlab
@raw_data = gollum_page.raw_data
@name = gollum_page.name
@historical = gollum_page.historical?
+ @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page)
@version = version
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 4507ea923b4..6bd256f57c7 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -1,6 +1,8 @@
require 'base64'
require 'gitaly'
+require 'grpc/health/v1/health_pb'
+require 'grpc/health/v1/health_services_pb'
module Gitlab
module GitalyClient
@@ -69,14 +71,27 @@ module Gitlab
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
- klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
- addr = address(storage)
- addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+ klass = stub_class(name)
+ addr = stub_address(storage)
klass.new(addr, :this_channel_is_insecure)
end
end
end
+ def self.stub_class(name)
+ if name == :health_check
+ Grpc::Health::V1::Health::Stub
+ else
+ Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
+ end
+ end
+
+ def self.stub_address(storage)
+ addr = address(storage)
+ addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+ addr
+ end
+
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index a250eb75bd4..ee36684197b 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -32,6 +32,26 @@ module Gitlab
binary: Gitlab::Git::Blob.binary?(data)
)
end
+
+ def batch_lfs_pointers(blob_ids)
+ request = Gitaly::GetLFSPointersRequest.new(
+ repository: @gitaly_repo,
+ blob_ids: blob_ids
+ )
+
+ response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request)
+
+ response.flat_map do |message|
+ message.lfs_pointers.map do |lfs_pointer|
+ Gitlab::Git::Blob.new(
+ id: lfs_pointer.oid,
+ size: lfs_pointer.size,
+ data: lfs_pointer.data,
+ binary: Gitlab::Git::Blob.binary?(lfs_pointer.data)
+ )
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb
new file mode 100644
index 00000000000..6c1213f5e20
--- /dev/null
+++ b/lib/gitlab/gitaly_client/health_check_service.rb
@@ -0,0 +1,19 @@
+module Gitlab
+ module GitalyClient
+ class HealthCheckService
+ def initialize(storage)
+ @storage = storage
+ end
+
+ # Sends a gRPC health ping to the Gitaly server for the storage shard.
+ def check
+ request = Grpc::Health::V1::HealthCheckRequest.new
+ response = GitalyClient.call(@storage, :health_check, :check, request, timeout: GitalyClient.fast_timeout)
+
+ { success: response&.status == :SERVING }
+ rescue GRPC::BadStatus => e
+ { success: false, message: e.to_s }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 12016aee2a6..654a3c314f1 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -161,6 +161,23 @@ module Gitlab
return response.error.b, 1
end
end
+
+ def create_bundle(save_path)
+ request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
+ response = GitalyClient.call(
+ @storage,
+ :repository_service,
+ :create_bundle,
+ request,
+ timeout: GitalyClient.default_timeout
+ )
+
+ File.open(save_path, 'wb') do |f|
+ response.each do |message|
+ f.write(message.data)
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
new file mode 100644
index 00000000000..11416c002e3
--- /dev/null
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -0,0 +1,53 @@
+module Gitlab
+ module HealthChecks
+ class GitalyCheck
+ extend BaseAbstractCheck
+
+ METRIC_PREFIX = 'gitaly_health_check'.freeze
+
+ class << self
+ def readiness
+ repository_storages.map do |storage_name|
+ check(storage_name)
+ end
+ end
+
+ def metrics
+ repository_storages.flat_map do |storage_name|
+ result, elapsed = with_timing { check(storage_name) }
+ labels = { shard: storage_name }
+
+ [
+ metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels),
+ metric("#{metric_prefix}_latency_seconds", elapsed, **labels)
+ ].flatten
+ end
+ end
+
+ def check(storage_name)
+ serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name)
+ result = serv.check
+ HealthChecks::Result.new(result[:success], result[:message], shard: storage_name)
+ end
+
+ private
+
+ def metric_prefix
+ METRIC_PREFIX
+ end
+
+ def successful?(result)
+ result[:success]
+ end
+
+ def repository_storages
+ storages.keys
+ end
+
+ def storages
+ Gitlab.config.repositories.storages
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index dd5d35feab9..25399f307f2 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -11,10 +11,6 @@ module Gitlab
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
- def git_bundle(repo_path:, bundle_path:)
- execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
- end
-
def git_clone_bundle(repo_path:, bundle_path:)
execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path}))
Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index a7028a32570..695462c7dd2 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -21,7 +21,7 @@ module Gitlab
def bundle_to_disk
mkdir_p(@shared.export_path)
- git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+ @project.repository.bundle_to_disk(@full_path)
rescue => e
@shared.error(e)
false
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 1e6722a7bba..5fa2e101e29 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -10,7 +10,7 @@ module Gitlab
def bundle_to_disk(full_path)
mkdir_p(@shared.export_path)
- git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+ @wiki.repository.bundle_to_disk(full_path)
rescue => e
@shared.error(e)
false
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 7771b15069b..4823f703ba4 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -20,7 +20,7 @@ module Gitlab
when 'commits'
Kaminari.paginate_array(commits).page(page).per(per_page)
else
- super
+ super(scope, page, false)
end
end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 70b639501fd..7362514167f 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -40,19 +40,21 @@ module Gitlab
@default_project_filter = default_project_filter
end
- def objects(scope, page = nil)
- case scope
- when 'projects'
- projects.page(page).per(per_page)
- when 'issues'
- issues.page(page).per(per_page)
- when 'merge_requests'
- merge_requests.page(page).per(per_page)
- when 'milestones'
- milestones.page(page).per(per_page)
- else
- Kaminari.paginate_array([]).page(page).per(per_page)
- end
+ def objects(scope, page = nil, without_count = true)
+ collection = case scope
+ when 'projects'
+ projects.page(page).per(per_page)
+ when 'issues'
+ issues.page(page).per(per_page)
+ when 'merge_requests'
+ merge_requests.page(page).per(per_page)
+ when 'milestones'
+ milestones.page(page).per(per_page)
+ else
+ Kaminari.paginate_array([]).page(page).per(per_page)
+ end
+
+ without_count ? collection.without_count : collection
end
def projects_count
@@ -71,18 +73,46 @@ module Gitlab
@milestones_count ||= milestones.count
end
+ def limited_projects_count
+ @limited_projects_count ||= projects.limit(count_limit).count
+ end
+
+ def limited_issues_count
+ return @limited_issues_count if @limited_issues_count
+
+ # By default getting limited count (e.g. 1000+) is fast on issuable
+ # collections except for issues, where filtering both not confidential
+ # and confidential issues user has access to, is too complex.
+ # It's faster to try to fetch all public issues first, then only
+ # if necessary try to fetch all issues.
+ sum = issues(public_only: true).limit(count_limit).count
+ @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum
+ end
+
+ def limited_merge_requests_count
+ @limited_merge_requests_count ||= merge_requests.limit(count_limit).count
+ end
+
+ def limited_milestones_count
+ @limited_milestones_count ||= milestones.limit(count_limit).count
+ end
+
def single_commit_result?
false
end
+ def count_limit
+ 1001
+ end
+
private
def projects
limit_projects.search(query)
end
- def issues
- issues = IssuesFinder.new(current_user).execute
+ def issues(finder_params = {})
+ issues = IssuesFinder.new(current_user, finder_params).execute
unless default_project_filter
issues = issues.where(project_id: project_ids_relation)
end
@@ -94,13 +124,13 @@ module Gitlab
issues.full_search(query)
end
- issues.order('updated_at DESC')
+ issues.reorder('updated_at DESC')
end
def milestones
milestones = Milestone.where(project_id: project_ids_relation)
milestones = milestones.search(query)
- milestones.order('updated_at DESC')
+ milestones.reorder('updated_at DESC')
end
def merge_requests
@@ -116,7 +146,7 @@ module Gitlab
merge_requests.full_search(query)
end
- merge_requests.order('updated_at DESC')
+ merge_requests.reorder('updated_at DESC')
end
def default_scope
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index b85f70e450e..4f86b3e8f73 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -16,7 +16,7 @@ module Gitlab
when 'snippet_blobs'
snippet_blobs.page(page).per(per_page)
else
- super
+ super(scope, nil, false)
end
end
View Lorry Controller Status