summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/gitlab/git.rake34
-rw-r--r--lib/tasks/gitlab/graphql.rake42
-rw-r--r--lib/tasks/gitlab/packages/events.rake34
-rw-r--r--lib/tasks/gitlab/pages.rake35
-rw-r--r--lib/tasks/gitlab/storage.rake15
5 files changed, 144 insertions, 16 deletions
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index 8a53b51d4fe..abb15f29328 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -21,5 +21,39 @@ namespace :gitlab do
failures.each { |f| puts "- #{f}" }
end
end
+
+ # Example for all projects:
+ #
+ # $ bin/rake gitlab:git:checksum_projects
+ # 1,cfa3f06ba235c13df0bb28e079bcea62c5848af2
+ # 2,
+ # 3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87
+ # 4,0000000000000000000000000000000000000000
+ #
+ # Example with a list of project IDs:
+ #
+ # $ CHECKSUM_PROJECT_IDS="1,3" bin/rake gitlab:git:checksum_projects
+ # 1,cfa3f06ba235c13df0bb28e079bcea62c5848af2
+ # 3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87
+ #
+ # - If a repository does not exist, the project ID is output with a blank checksum
+ # - If a repository exists but is empty, the output checksum is `0000000000000000000000000000000000000000`
+ # - If given specific IDs, projects which do not exist are skipped
+ desc 'GitLab | Git | Generate checksum of project repository refs'
+ task checksum_projects: :environment do
+ project_ids = ENV['CHECKSUM_PROJECT_IDS']&.split(',')
+ relation = Project
+ relation = relation.where(id: project_ids) if project_ids.present?
+
+ relation.find_each(batch_size: 100) do |project|
+ next unless project.repo_exists?
+
+ result = project.repository.checksum
+ rescue => e
+ result = "Ignored error: #{e.message}".squish.truncate(255)
+ ensure
+ puts "#{project.id},#{result}"
+ end
+ end
end
end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index 5a583183924..f708114c226 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -33,6 +33,44 @@ namespace :gitlab do
)
namespace :graphql do
+ desc 'Gitlab | GraphQL | Validate queries'
+ task validate: [:environment, :enable_feature_flags] do |t, args|
+ queries = if args.to_a.present?
+ args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) }
+ else
+ Gitlab::Graphql::Queries.all
+ end
+
+ failed = queries.flat_map do |defn|
+ summary, errs = defn.validate(GitlabSchema)
+
+ case summary
+ when :client_query
+ warn("SKIP #{defn.file}: client query")
+ else
+ warn("OK #{defn.file}") if errs.empty?
+ errs.each do |err|
+ warn(<<~MSG)
+ ERROR #{defn.file}: #{err.message} (at #{err.path.join('.')})
+ MSG
+ end
+ end
+
+ errs.empty? ? [] : [defn.file]
+ end
+
+ if failed.present?
+ format_output(
+ "#{failed.count} GraphQL #{'query'.pluralize(failed.count)} out of #{queries.count} failed validation:",
+ *failed.map do |name|
+ known_failure = Gitlab::Graphql::Queries.known_failure?(name)
+ "- #{name}" + (known_failure ? ' (known failure)' : '')
+ end
+ )
+ abort unless failed.all? { |name| Gitlab::Graphql::Queries.known_failure?(name) }
+ end
+ end
+
desc 'GitLab | GraphQL | Generate GraphQL docs'
task compile_docs: [:environment, :enable_feature_flags] do
renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
@@ -78,11 +116,11 @@ def render_options
}
end
-def format_output(str)
+def format_output(*strs)
heading = '#' * 10
puts heading
puts '#'
- puts "# #{str}"
+ strs.each { |str| puts "# #{str}" }
puts '#'
puts heading
end
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
index ca507fb5320..cfe97984dda 100644
--- a/lib/tasks/gitlab/packages/events.rake
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -5,18 +5,18 @@ namespace :gitlab do
namespace :packages do
namespace :events do
task generate: :environment do
- Rake::Task["gitlab:packages:events:generate_guest"].invoke
+ Rake::Task["gitlab:packages:events:generate_counts"].invoke
Rake::Task["gitlab:packages:events:generate_unique"].invoke
rescue => e
logger.error("Error building events list: #{e}")
end
- task generate_guest: :environment do
+ task generate_counts: :environment do
logger = Logger.new(STDOUT)
logger.info('Building list of package events...')
- path = Gitlab::UsageDataCounters::GuestPackageEventCounter::KNOWN_EVENTS_PATH
- File.open(path, "w") { |file| file << guest_events_list.to_yaml }
+ path = Gitlab::UsageDataCounters::PackageEventCounter::KNOWN_EVENTS_PATH
+ File.open(path, "w") { |file| file << counter_events_list.to_yaml }
logger.info("Events file `#{path}` generated successfully")
rescue => e
@@ -43,26 +43,32 @@ namespace :gitlab do
def generate_unique_events_list
events = event_pairs.each_with_object([]) do |(event_type, event_scope), events|
- Packages::Event.originator_types.keys.excluding('guest').each do |originator|
- if name = Packages::Event.allowed_event_name(event_scope, event_type, originator)
- events << {
- "name" => name,
- "category" => "#{event_scope}_packages",
+ Packages::Event.originator_types.keys.excluding('guest').each do |originator_type|
+ events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name|
+ {
+ "name" => event_name,
+ "category" => "#{originator_type}_packages",
"aggregation" => "weekly",
"redis_slot" => "package",
"feature_flag" => "collect_package_events_redis"
}
end
+
+ events.concat(events_definition)
end
end
- events.sort_by { |event| event["name"] }
+ events.sort_by { |event| event["name"] }.uniq
end
- def guest_events_list
- event_pairs.map do |event_type, event_scope|
- Packages::Event.allowed_event_name(event_scope, event_type, "guest")
- end.compact.sort
+ def counter_events_list
+ counters = event_pairs.flat_map do |event_type, event_scope|
+ Packages::Event.originator_types.keys.flat_map do |originator_type|
+ Packages::Event.counters_for(event_scope, event_type, originator_type)
+ end
+ end
+
+ counters.compact.sort.uniq
end
end
end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
new file mode 100644
index 00000000000..e15cbb4e32e
--- /dev/null
+++ b/lib/tasks/gitlab/pages.rake
@@ -0,0 +1,35 @@
+require 'logger'
+
+namespace :gitlab do
+ namespace :pages do
+ desc "GitLab | Pages | Migrate legacy storage to zip format"
+ task migrate_legacy_storage: :gitlab_environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting to migrate legacy pages storage to zip deployments')
+ processed_projects = 0
+
+ ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch|
+ batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum|
+ project = metadatum.project
+
+ result = nil
+ time = Benchmark.realtime do
+ result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
+ end
+ processed_projects += 1
+
+ if result[:status] == :success
+ logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
+ else
+ logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}")
+ end
+ rescue => e
+ logger.error("#{e.message} project_id: #{project&.id}")
+ Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
+ end
+
+ logger.info("#{processed_projects} pages projects are processed")
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index ccc96b7edfb..f7819fd974b 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -116,6 +116,21 @@ namespace :gitlab do
helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
end
+ desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated'
+ task prune_hashed_projects: [:environment, :gitlab_environment] do
+ if Rails.env.production?
+ abort('This destructive action may only be run in development')
+ end
+
+ helper = Gitlab::HashedStorage::RakeHelper
+ name = 'projects using Hashed Storage'
+ relation = Project.with_storage_feature(:repository)
+ root = Gitlab.config.repositories.storages['default'].legacy_disk_path
+ dry_run = !ENV['FORCE'].present?
+
+ helper.prune(name, relation, dry_run: dry_run, root: root)
+ end
+
desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
task legacy_attachments: :environment do
helper = Gitlab::HashedStorage::RakeHelper
View Lorry Controller Status