1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
namespace :gitlab do
desc "GitLab | Update templates"
task :update_templates do
TEMPLATE_DATA.each { |template| update(template) }
end
desc "GitLab | Update project templates"
task :update_project_templates, [] => :environment do |_task, args|
# we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't
# want to include it in the task, as this would affect subsequent tasks as well
downloader = Class.new do
extend Gitlab::ImportExport::CommandLineUtil
def self.call(uploader, upload_path)
download_or_copy_upload(uploader, upload_path)
end
end
template_names = args.extras.to_set
if Rails.env.production?
raise "This rake task is not meant for production instances"
end
admin = User.find_by(admin: true)
unless admin
raise "No admin user could be found"
end
tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}"
puts "Creating temporary namespace #{tmp_namespace_path}"
tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path)
templates = if template_names.empty?
Gitlab::ProjectTemplate.all
else
Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) }
end
templates.each do |template|
params = {
namespace_id: tmp_namespace.id,
path: template.name,
skip_wiki: true
}
puts "Creating project for #{template.title}"
project = Projects::CreateService.new(admin, params).execute
unless project.persisted?
raise "Failed to create project: #{project.errors.messages}"
end
uri_encoded_project_path = template.uri_encoded_project_path
# extract a concrete commit for signing off what we actually downloaded
# this way we do the right thing even if the repository gets updated in the meantime
get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits",
query: { page: 1, per_page: 1 }
)
raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success?
commit_sha = get_commits_response.parsed_response.dig(0, 'id')
project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}"
commit_message = <<~MSG
Initialized from '#{template.title}' project template
Template repository: #{template.preview}
Commit SHA: #{commit_sha}
MSG
Dir.mktmpdir do |tmpdir|
Dir.chdir(tmpdir) do
Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz'])
Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz'])
extracted_project_basename = Dir['*/'].first
Dir.chdir(extracted_project_basename) do
Gitlab::TaskHelpers.run_command!(%w(git init))
Gitlab::TaskHelpers.run_command!(%w(git add .))
Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message])
# Hacky workaround to push to the project in a way that works with both GDK and the test environment
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"])
end
Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master'])
end
end
end
project.reset
Projects::ImportExport::ExportService.new(project, admin).execute
downloader.call(project.export_file, template.archive_path)
unless Projects::DestroyService.new(project, admin).execute
puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)"
end
puts "Exported #{template.name}".green
end
success = true
ensure
if tmp_namespace
puts "Destroying temporary namespace #{tmp_namespace_path}"
tmp_namespace.destroy
end
puts "Done".green if success
end
def update(template)
sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
dir = File.join(vendor_directory, sub_dir)
unless clone_repository(template.repo_url, dir)
puts "Cloning the #{sub_dir} templates failed".red
return
end
remove_unneeded_files(dir, template.cleanup_regex)
puts "Done".green
end
def clone_repository(url, directory)
FileUtils.rm_rf(directory) if Dir.exist?(directory)
system("git clone #{url} --depth=1 --branch=master #{directory}")
end
# Retain only certain files:
# - The LICENSE, because we have to
# - The sub dirs so we can organise the file by category
# - The templates themself
# - Dir.entries returns also the entries '.' and '..'
def remove_unneeded_files(directory, regex)
Dir.foreach(directory) do |file|
FileUtils.rm_rf(File.join(directory, file)) unless file =~ regex
end
end
private
Template = Struct.new(:repo_url, :cleanup_regex)
TEMPLATE_DATA = [
Template.new(
"https://github.com/github/gitignore.git",
/(\.{1,2}|LICENSE|Global|\.gitignore)\z/
),
Template.new(
"https://gitlab.com/gitlab-org/Dockerfile.git",
/(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
)
].freeze
def vendor_directory
Rails.root.join('vendor')
end
end
|
