summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/brakeman.rake2
-rw-r--r--lib/tasks/gitlab/cleanup.rake7
-rw-r--r--lib/tasks/gitlab/gitaly.rake2
-rw-r--r--lib/tasks/gitlab/storage.rake85
4 files changed, 78 insertions, 18 deletions
diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index 99b3168d9eb..2301ec9b228 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
task :brakeman do
# We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
# requests are welcome!
- if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z))
+ if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
puts 'Security check succeed'
else
puts 'Security check failed'
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 301affc9522..eb0f757aea7 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,11 +1,14 @@
namespace :gitlab do
namespace :cleanup do
+ HASHED_REPOSITORY_NAME = '@hashed'.freeze
+
desc "GitLab | Cleanup | Clean namespaces"
task dirs: :environment do
warn_user_is_not_gitlab
remove_flag = ENV['REMOVE']
- namespaces = Namespace.pluck(:path)
+ namespaces = Namespace.pluck(:path)
+ namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored
Gitlab.config.repositories.storages.each do |name, repository_storage|
git_base_path = repository_storage['path']
all_dirs = Dir.glob(git_base_path + '/*')
@@ -62,7 +65,7 @@ namespace :gitlab do
# TODO ignoring hashed repositories for now. But revisit to fully support
# possible orphaned hashed repos
- next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace)
+ next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
new_path = path + move_suffix
puts path.inspect + ' -> ' + new_path.inspect
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index f2002d7a426..4d880c05f99 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -78,6 +78,8 @@ namespace :gitlab do
config[:auth] = { token: 'secret' } if Rails.env.test?
config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
+ config[:bin_dir] = Gitlab.config.gitaly.client_path
+
TOML.dump(config)
end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index e05be4a3405..8ac73bc8ff2 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,10 +2,10 @@ namespace :gitlab do
namespace :storage do
desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
task migrate_to_hashed: :environment do
- legacy_projects_count = Project.with_legacy_storage.count
+ legacy_projects_count = Project.with_unmigrated_storage.count
if legacy_projects_count == 0
- puts 'There are no projects using legacy storage. Nothing to do!'
+ puts 'There are no projects requiring storage migration. Nothing to do!'
next
end
@@ -23,22 +23,42 @@ namespace :gitlab do
desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
task legacy_projects: :environment do
- projects_summary(Project.with_legacy_storage)
+ relation_summary('projects', Project.without_storage_feature(:repository))
end
desc 'Gitlab | Storage | List existing projects using Legacy Storage'
task list_legacy_projects: :environment do
- projects_list(Project.with_legacy_storage)
+ projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
end
desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
task hashed_projects: :environment do
- projects_summary(Project.with_hashed_storage)
+ relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
end
desc 'Gitlab | Storage | List existing projects using Hashed Storage'
task list_hashed_projects: :environment do
- projects_list(Project.with_hashed_storage)
+ projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+ end
+
+ desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
+ task legacy_attachments: :environment do
+ relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+ end
+
+ desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
+ task list_legacy_attachments: :environment do
+ attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+ end
+
+ desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
+ task hashed_attachments: :environment do
+ relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+ end
+
+ desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
+ task list_hashed_attachments: :environment do
+ attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
end
def batch_size
@@ -46,29 +66,43 @@ namespace :gitlab do
end
def project_id_batches(&block)
- Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+ Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
ids = relation.pluck(:id)
yield ids.min, ids.max
end
end
- def projects_summary(relation)
- projects_count = relation.count
- puts "* Found #{projects_count} projects".color(:green)
+ def legacy_attachments_relation
+ Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+ JOIN projects
+ ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+ SQL
+ end
+
+ def hashed_attachments_relation
+ Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+ JOIN projects
+ ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+ SQL
+ end
+
+ def relation_summary(relation_name, relation)
+ relation_count = relation.count
+ puts "* Found #{relation_count} #{relation_name}".color(:green)
- projects_count
+ relation_count
end
- def projects_list(relation)
- projects_count = projects_summary(relation)
+ def projects_list(relation_name, relation)
+ relation_count = relation_summary(relation_name, relation)
projects = relation.with_route
limit = ENV.fetch('LIMIT', 500).to_i
- return unless projects_count > 0
+ return unless relation_count > 0
- puts " ! Displaying first #{limit} projects..." if projects_count > limit
+ puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
counter = 0
projects.find_in_batches(batch_size: batch_size) do |batch|
@@ -81,5 +115,26 @@ namespace :gitlab do
end
end
end
+
+ def attachments_list(relation_name, relation)
+ relation_count = relation_summary(relation_name, relation)
+
+ limit = ENV.fetch('LIMIT', 500).to_i
+
+ return unless relation_count > 0
+
+ puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
+
+ counter = 0
+ relation.find_in_batches(batch_size: batch_size) do |batch|
+ batch.each do |upload|
+ counter += 1
+
+ puts " - #{upload.path} (id: #{upload.id})".color(:red)
+
+ return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+ end
+ end
+ end
end
end
View Lorry Controller Status