summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers.rb3
-rw-r--r--lib/api/markdown.rb4
-rw-r--r--lib/banzai/filter/markdown_filter.rb2
-rw-r--r--lib/gitlab/gitaly_client.rb14
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb2
-rw-r--r--lib/gitlab/github_import/parallel_importer.rb9
-rw-r--r--lib/gitlab/verify/batch_verifier.rb59
-rw-r--r--lib/gitlab/verify/job_artifacts.rb10
-rw-r--r--lib/gitlab/verify/lfs_objects.rb12
-rw-r--r--lib/gitlab/verify/rake_task.rb2
-rw-r--r--lib/gitlab/verify/uploads.rb12
-rw-r--r--lib/system_check/orphans/repository_check.rb16
-rw-r--r--lib/system_check/simple_executor.rb30
-rw-r--r--lib/tasks/migrate/setup_postgresql.rake15
14 files changed, 151 insertions, 39 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 2ed331d4fd2..9c53b7c3fe7 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -272,7 +272,8 @@ module API
attrs[key] = params_hash[key]
end
end
- ActionController::Parameters.new(attrs).permit!
+ permitted_attrs = ActionController::Parameters.new(attrs).permit!
+ Gitlab.rails5? ? permitted_attrs.to_h : permitted_attrs
end
def filter_by_iid(items, iid)
diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb
index b9ed68aa584..5d55224c1a7 100644
--- a/lib/api/markdown.rb
+++ b/lib/api/markdown.rb
@@ -10,9 +10,7 @@ module API
detail "This feature was introduced in GitLab 11.0."
end
post do
- # Explicitly set CommonMark as markdown engine to use.
- # Remove this set when https://gitlab.com/gitlab-org/gitlab-ce/issues/43011 is done.
- context = { markdown_engine: :common_mark, only_path: false }
+ context = { only_path: false }
if params[:project]
project = Project.find_by_full_path(params[:project])
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index c1e2b680240..944363f17d3 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -14,7 +14,7 @@ module Banzai
private
- DEFAULT_ENGINE = :redcarpet
+ DEFAULT_ENGINE = :common_mark
def engine(engine_from_context)
engine_from_context ||= DEFAULT_ENGINE
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 36e9adf27da..620362b52a9 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -33,11 +33,6 @@ module Gitlab
MAXIMUM_GITALY_CALLS = 35
CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
- # We have a mechanism to let GitLab automatically opt in to all Gitaly
- # features. We want to be able to exclude some features from automatic
- # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for.
- EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze
-
MUTEX = Mutex.new
class << self
@@ -249,7 +244,7 @@ module Gitlab
when MigrationStatus::OPT_OUT
true
when MigrationStatus::OPT_IN
- opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name)
+ opt_into_all_features? && !explicit_opt_in_required.include?(feature_name)
else
false
end
@@ -259,6 +254,13 @@ module Gitlab
false
end
+ # We have a mechanism to let GitLab automatically opt in to all Gitaly
+ # features. We want to be able to exclude some features from automatic
+ # opt-in. This function has an override in EE.
+ def self.explicit_opt_in_required
+ []
+ end
+
# opt_into_all_features? returns true when the current environment
# is one in which we opt into features automatically
def self.opt_into_all_features?
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a4cc64de80d..7f2e6441f16 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -179,6 +179,8 @@ module Gitlab
end
def list_commits_by_oid(oids)
+ return [] if oids.empty?
+
request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids)
response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb
index b02b123c98e..a77ac1e4fa6 100644
--- a/lib/gitlab/github_import/parallel_importer.rb
+++ b/lib/gitlab/github_import/parallel_importer.rb
@@ -15,6 +15,15 @@ module Gitlab
true
end
+ # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore
+ # the visibility of prepended modules. See
+ # https://github.com/rspec/rspec-mocks/issues/1231 for more details.
+ if Rails.env.test?
+ def self.requires_ci_cd_setup?
+ raise NotImplementedError
+ end
+ end
+
def initialize(project)
@project = project
end
diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb
index 1ef369a4b67..167ba1b3149 100644
--- a/lib/gitlab/verify/batch_verifier.rb
+++ b/lib/gitlab/verify/batch_verifier.rb
@@ -7,13 +7,15 @@ module Gitlab
@batch_size = batch_size
@start = start
@finish = finish
+
+ fix_google_api_logger
end
# Yields a Range of IDs and a Hash of failed verifications (object => error)
def run_batches(&blk)
- relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches
- range = relation.first.id..relation.last.id
- failures = run_batch(relation)
+ all_relation.in_batches(of: batch_size, start: start, finish: finish) do |batch| # rubocop: disable Cop/InBatches
+ range = batch.first.id..batch.last.id
+ failures = run_batch_for(batch)
yield(range, failures)
end
@@ -29,24 +31,56 @@ module Gitlab
private
- def run_batch(relation)
- relation.map { |upload| verify(upload) }.compact.to_h
+ def run_batch_for(batch)
+ batch.map { |upload| verify(upload) }.compact.to_h
end
def verify(object)
+ local?(object) ? verify_local(object) : verify_remote(object)
+ rescue => err
+ failure(object, err.inspect)
+ end
+
+ def verify_local(object)
expected = expected_checksum(object)
actual = actual_checksum(object)
- raise 'Checksum missing' unless expected.present?
- raise 'Checksum mismatch' unless expected == actual
+ return failure(object, 'Checksum missing') unless expected.present?
+ return failure(object, 'Checksum mismatch') unless expected == actual
+
+ success
+ end
+ # We don't calculate checksum for remote objects, so just check existence
+ def verify_remote(object)
+ return failure(object, 'Remote object does not exist') unless remote_object_exists?(object)
+
+ success
+ end
+
+ def success
nil
- rescue => err
- [object, err]
+ end
+
+ def failure(object, message)
+ [object, message]
+ end
+
+ # It's already set to Logger::INFO, but acts as if it is set to
+ # Logger::DEBUG, and this fixes it...
+ def fix_google_api_logger
+ if Object.const_defined?('Google::Apis')
+ Google::Apis.logger.level = Logger::INFO
+ end
end
# This should return an ActiveRecord::Relation suitable for calling #in_batches on
- def relation
+ def all_relation
+ raise NotImplementedError.new
+ end
+
+ # Should return true if the object is stored locally
+ def local?(_object)
raise NotImplementedError.new
end
@@ -59,6 +93,11 @@ module Gitlab
def actual_checksum(_object)
raise NotImplementedError.new
end
+
+ # Be sure to perform a hard check of the remote object (don't just check DB value)
+ def remote_object_exists?(object)
+ raise NotImplementedError.new
+ end
end
end
end
diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb
index 03500a61074..dbadfbde9e3 100644
--- a/lib/gitlab/verify/job_artifacts.rb
+++ b/lib/gitlab/verify/job_artifacts.rb
@@ -11,10 +11,14 @@ module Gitlab
private
- def relation
+ def all_relation
::Ci::JobArtifact.all
end
+ def local?(artifact)
+ artifact.local_store?
+ end
+
def expected_checksum(artifact)
artifact.file_sha256
end
@@ -22,6 +26,10 @@ module Gitlab
def actual_checksum(artifact)
Digest::SHA256.file(artifact.file.path).hexdigest
end
+
+ def remote_object_exists?(artifact)
+ artifact.file.file.exists?
+ end
end
end
end
diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb
index 970e2a7b718..d3f58a73ac7 100644
--- a/lib/gitlab/verify/lfs_objects.rb
+++ b/lib/gitlab/verify/lfs_objects.rb
@@ -11,8 +11,12 @@ module Gitlab
private
- def relation
- LfsObject.with_files_stored_locally
+ def all_relation
+ LfsObject.all
+ end
+
+ def local?(lfs_object)
+ lfs_object.local_store?
end
def expected_checksum(lfs_object)
@@ -22,6 +26,10 @@ module Gitlab
def actual_checksum(lfs_object)
LfsObject.calculate_oid(lfs_object.file.path)
end
+
+ def remote_object_exists?(lfs_object)
+ lfs_object.file.file.exists?
+ end
end
end
end
diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb
index dd138e6b92b..e190eaddc79 100644
--- a/lib/gitlab/verify/rake_task.rb
+++ b/lib/gitlab/verify/rake_task.rb
@@ -45,7 +45,7 @@ module Gitlab
return unless verbose?
failures.each do |object, error|
- say " - #{verifier.describe(object)}: #{error.inspect}".color(:red)
+ say " - #{verifier.describe(object)}: #{error}".color(:red)
end
end
end
diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb
index 0ffa71a6d72..01f09ab8df7 100644
--- a/lib/gitlab/verify/uploads.rb
+++ b/lib/gitlab/verify/uploads.rb
@@ -11,8 +11,12 @@ module Gitlab
private
- def relation
- Upload.with_files_stored_locally
+ def all_relation
+ Upload.all
+ end
+
+ def local?(upload)
+ upload.local?
end
def expected_checksum(upload)
@@ -22,6 +26,10 @@ module Gitlab
def actual_checksum(upload)
Upload.hexdigest(upload.absolute_path)
end
+
+ def remote_object_exists?(upload)
+ upload.build_uploader.file.exists?
+ end
end
end
end
diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb
index 5ef0b93ad08..2695c658874 100644
--- a/lib/system_check/orphans/repository_check.rb
+++ b/lib/system_check/orphans/repository_check.rb
@@ -5,16 +5,18 @@ module SystemCheck
attr_accessor :orphans
def multi_check
- Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
- storage_path = repository_storage.legacy_disk_path
+ Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+ Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+ storage_path = repository_storage.legacy_disk_path
- $stdout.puts
- $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow)
+ $stdout.puts
+ $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow)
- repositories = disk_repositories(storage_path)
- orphans = (repositories - fetch_repositories(storage_name))
+ repositories = disk_repositories(storage_path)
+ orphans = (repositories - fetch_repositories(storage_name))
- print_orphans(orphans, storage_name)
+ print_orphans(orphans, storage_name)
+ end
end
end
diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb
index d268f501b4a..99c9e984107 100644
--- a/lib/system_check/simple_executor.rb
+++ b/lib/system_check/simple_executor.rb
@@ -43,7 +43,7 @@ module SystemCheck
#
# @param [SystemCheck::BaseCheck] check_klass
def run_check(check_klass)
- $stdout.print "#{check_klass.display_name} ... "
+ print_display_name(check_klass)
check = check_klass.new
@@ -60,18 +60,18 @@ module SystemCheck
end
if check.check?
- $stdout.puts check_klass.check_pass.color(:green)
+ print_check_pass(check_klass)
else
- $stdout.puts check_klass.check_fail.color(:red)
+ print_check_failure(check_klass)
if check.can_repair?
$stdout.print 'Trying to fix error automatically. ...'
if check.repair!
- $stdout.puts 'Success'.color(:green)
+ print_success
return
else
- $stdout.puts 'Failed'.color(:red)
+ print_failure
end
end
@@ -83,6 +83,26 @@ module SystemCheck
private
+ def print_display_name(check_klass)
+ $stdout.print "#{check_klass.display_name} ... "
+ end
+
+ def print_check_pass(check_klass)
+ $stdout.puts check_klass.check_pass.color(:green)
+ end
+
+ def print_check_failure(check_klass)
+ $stdout.puts check_klass.check_fail.color(:red)
+ end
+
+ def print_success
+ $stdout.puts 'Success'.color(:green)
+ end
+
+ def print_failure
+ $stdout.puts 'Failed'.color(:red)
+ end
+
# Prints header content for the series of checks to be executed for this component
#
# @param [String] component name of the component relative to the checks being executed
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index e7aab50e42a..f69d204c579 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -22,3 +22,18 @@ task setup_postgresql: :environment do
ProjectNameLowerIndex.new.up
AddPathIndexToRedirectRoutes.new.up
end
+
+desc 'GitLab | Generate PostgreSQL Password Hash'
+task :postgresql_md5_hash do
+ require 'digest'
+ username = ENV.fetch('USERNAME') do |missing|
+ puts "You must provide an username with '#{missing}' ENV variable"
+ exit(1)
+ end
+ password = ENV.fetch('PASSWORD') do |missing|
+ puts "You must provide a password with '#{missing}' ENV variable"
+ exit(1)
+ end
+ hash = Digest::MD5.hexdigest("#{password}#{username}")
+ puts "The MD5 hash of your database password for user: #{username} -> #{hash}"
+end
View Lorry Controller Status