1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
|
# frozen_string_literal: true
require 'parallel'
require_relative 'gitaly_setup'
require_relative '../../../lib/gitlab/setup_helper'
module TestEnv
extend self
ComponentFailedToInstallError = Class.new(StandardError)
# When developing the seed repository, comment out the branch you will modify.
BRANCH_SHA = {
'signed-commits' => 'c7794c1',
'gpg-signed' => '8a852d5',
'x509-signed' => 'a4df3c8',
'not-merged-branch' => 'b83d6e3',
'branch-merged' => '498214d',
'empty-branch' => '7efb185',
'ends-with.json' => '98b0d8b',
'flatten-dir' => 'e56497b',
'feature' => '0b4bc9a',
'feature_conflict' => 'bb5206f',
'fix' => '48f0be4',
'improve/awesome' => '5937ac0',
'merged-target' => '21751bf',
'markdown' => '0ed8c6c',
'lfs' => '55bc176',
'master' => 'b83d6e391c22777fca1ed3012fce84f633d7fed0',
'merge-test' => '5937ac0',
"'test'" => 'e56497b',
'orphaned-branch' => '45127a9',
'binary-encoding' => '7b1cf43',
'gitattributes' => '5a62481',
'expand-collapse-diffs' => '4842455',
'symlink-expand-diff' => '81e6355',
'diff-files-symlink-to-image' => '8cfca84',
'diff-files-image-to-symlink' => '3e94fda',
'diff-files-symlink-to-text' => '689815e',
'diff-files-text-to-symlink' => '5e2c270',
'expand-collapse-files' => '025db92',
'expand-collapse-lines' => '238e82d',
'pages-deploy' => '7897d5b',
'pages-deploy-target' => '7975be0',
'audio' => 'c3c21fd',
'video' => '8879059',
'crlf-diff' => '5938907',
'conflict-start' => '824be60',
'conflict-resolvable' => '1450cd639e0bc6721eb02800169e464f212cde06',
'conflict-binary-file' => '259a6fb',
'conflict-contains-conflict-markers' => '78a3086',
'conflict-missing-side' => 'eb227b3',
'conflict-non-utf8' => 'd0a293c',
'conflict-too-large' => '39fa04f',
'deleted-image-test' => '6c17798',
'wip' => 'b9238ee',
'csv' => '3dd0896',
'v1.1.0' => 'b83d6e3',
'add-ipython-files' => '4963fef',
'add-pdf-file' => 'e774ebd',
'squash-large-files' => '54cec52',
'add-pdf-text-binary' => '79faa7b',
'add_images_and_changes' => '010d106',
'update-gitlab-shell-v-6-0-1' => '2f61d70',
'update-gitlab-shell-v-6-0-3' => 'de78448',
'merge-commit-analyze-before' => '1adbdef',
'merge-commit-analyze-side-branch' => '8a99451',
'merge-commit-analyze-after' => '646ece5',
'snippet/single-file' => '43e4080aaa14fc7d4b77ee1f5c9d067d5a7df10e',
'snippet/multiple-files' => '40232f7eb98b3f221886432def6e8bab2432add9',
'snippet/rename-and-edit-file' => '220a1e4b4dff37feea0625a7947a4c60fbe78365',
'snippet/edit-file' => 'c2f074f4f26929c92795a75775af79a6ed6d8430',
'snippet/no-files' => '671aaa842a4875e5f30082d1ab6feda345fdb94d',
'2-mb-file' => 'bf12d25',
'before-create-delete-modify-move' => '845009f',
'between-create-delete-modify-move' => '3f5f443',
'after-create-delete-modify-move' => 'ba3faa7',
'with-codeowners' => '219560e',
'submodule_inside_folder' => 'b491b92',
'png-lfs' => 'fe42f41',
'sha-starting-with-large-number' => '8426165',
'invalid-utf8-diff-paths' => '99e4853',
'compare-with-merge-head-source' => 'f20a03d',
'compare-with-merge-head-target' => '2f1e176',
'trailers' => 'f0a5ed6',
'add_commit_with_5mb_subject' => '8cf8e80',
'blame-on-renamed' => '32c33da',
'with-executables' => '6b8dc4a',
'spooky-stuff' => 'ba3343b',
'few-commits' => '0031876',
'two-commits' => '304d257',
'utf-16' => 'f05a987',
'gitaly-rename-test' => '94bb47c',
'smime-signed-commits' => 'ed775cc',
'Ääh-test-utf-8' => '7975be0',
'ssh-signed-commit' => '7b5160f'
}.freeze
# gitlab-test-fork is a fork of gitlab-fork, but we don't necessarily
# need to keep all the branches in sync.
# We currently only need a subset of the branches
FORKED_BRANCH_SHA = {
'add-submodule-version-bump' => '3f547c0',
'master' => '5937ac0',
'remove-submodule' => '2a33e0c',
'conflict-resolvable-fork' => '404fa3f'
}.freeze
TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
# Can be overriden
def setup_methods
SETUP_METHODS
end
# Test environment
#
# See gitlab.yml.example test section for paths
#
def init
unless Rails.env.test?
puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n"
exit 1
end
start = Time.now
# Disable mailer for spinach tests
clean_test_path
# Install components in parallel as most of the setup is I/O.
Parallel.each(setup_methods) do |method|
public_send(method)
end
post_init
puts "\nTest environment set up in #{Time.now - start} seconds"
end
# Can be overriden
def post_init
start_gitaly
end
# Clean /tmp/tests
#
# Keeps gitlab-shell and gitlab-test
def clean_test_path
Dir[File.join(TMP_TEST_PATH, '**')].each do |entry|
unless test_dirs.include?(File.basename(entry))
FileUtils.rm_rf(entry)
end
end
FileUtils.mkdir_p(
Gitlab::GitalyClient::StorageSettings.allow_disk_access { GitalySetup.repos_path }
)
FileUtils.mkdir_p(SECOND_STORAGE_PATH)
FileUtils.mkdir_p(backup_path)
FileUtils.mkdir_p(pages_path)
FileUtils.mkdir_p(artifacts_path)
FileUtils.mkdir_p(lfs_path)
FileUtils.mkdir_p(terraform_state_path)
FileUtils.mkdir_p(packages_path)
end
def setup_gitlab_shell
FileUtils.mkdir_p(Gitlab.config.gitlab_shell.path)
end
def setup_gitaly
component_timed_setup('Gitaly',
install_dir: GitalySetup.gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:clone",
fresh_install: ENV.key?('FORCE_GITALY_INSTALL'),
task_args: [GitalySetup.gitaly_dir, GitalySetup.repos_path, gitaly_url].compact) do
GitalySetup.setup_gitaly
end
end
def start_gitaly
if ci?
# Gitaly has been spawned outside this process already
return
end
GitalySetup.spawn_gitaly
end
def gitaly_url
ENV.fetch('GITALY_REPO_URL', nil)
end
# Feature specs are run through Workhorse
def setup_workhorse
# Always rebuild the config file
if skip_compile_workhorse?
Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil, force: true)
return
end
start = Time.now
FileUtils.rm_rf(workhorse_dir)
Gitlab::SetupHelper::Workhorse.compile_into(workhorse_dir)
Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil)
File.write(workhorse_tree_file, workhorse_tree) if workhorse_source_clean?
puts "==> GitLab Workhorse set up in #{Time.now - start} seconds...\n"
end
def skip_compile_workhorse?
File.directory?(workhorse_dir) &&
workhorse_source_clean? &&
File.exist?(workhorse_tree_file) &&
workhorse_tree == File.read(workhorse_tree_file)
end
def workhorse_source_clean?
out = IO.popen(%w[git status --porcelain workhorse], &:read)
$?.success? && out.empty?
end
def workhorse_tree
IO.popen(%w[git rev-parse HEAD:workhorse], &:read)
end
def workhorse_tree_file
File.join(workhorse_dir, 'WORKHORSE_TREE')
end
def workhorse_dir
@workhorse_path ||= Rails.root.join('tmp', 'tests', 'gitlab-workhorse')
end
def with_workhorse(host, port, upstream, &blk)
host = "[#{host}]" if host.include?(':')
listen_addr = [host, port].join(':')
config_path = Gitlab::SetupHelper::Workhorse.get_config_path(workhorse_dir, {})
workhorse_pid = spawn(
{ 'PATH' => "#{ENV['PATH']}:#{workhorse_dir}" },
File.join(workhorse_dir, 'gitlab-workhorse'),
'-authSocket', upstream,
'-documentRoot', Rails.root.join('public').to_s,
'-listenAddr', listen_addr,
'-secretPath', Gitlab::Workhorse.secret_path.to_s,
'-config', config_path,
'-logFile', 'log/workhorse-test.log',
'-logFormat', 'structured',
'-developmentMode' # to serve assets and rich error messages
)
begin
yield
ensure
Process.kill('TERM', workhorse_pid)
Process.wait(workhorse_pid)
end
end
def workhorse_url
ENV.fetch('GITLAB_WORKHORSE_URL', nil)
end
# Create repository for FactoryBot.create(:project)
def setup_factory_repo
setup_repo(factory_repo_path, factory_repo_bundle_path, factory_repo_name, BRANCH_SHA)
end
# Create repository for FactoryBot.create(:forked_project_with_submodules)
# This repo has a submodule commit that is not present in the main test
# repository.
def setup_forked_repo
setup_repo(forked_repo_path, forked_repo_bundle_path, forked_repo_name, FORKED_BRANCH_SHA)
end
def setup_repo(repo_path, repo_bundle_path, repo_name, refs)
clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
unless File.directory?(repo_path)
start = Time.now
system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path}))
puts "==> #{repo_path} set up in #{Time.now - start} seconds...\n"
end
create_bundle = !File.file?(repo_bundle_path)
unless set_repo_refs(repo_path, refs)
# Prefer not to fetch over the network. Only fetch when we have failed to
# set all the required local branches. This would happen when a new
# branch is added to BRANCH_SHA, in which case we want to update
# everything.
unless system(*%W(#{Gitlab.config.git.bin_path} -C #{repo_path} fetch origin))
raise 'Could not fetch test seed repository.'
end
unless set_repo_refs(repo_path, refs)
raise "Could not update test seed repository, please delete #{repo_path} and try again"
end
create_bundle = true
end
if create_bundle
start = Time.now
system(git_env, *%W(#{Gitlab.config.git.bin_path} -C #{repo_path} bundle create #{repo_bundle_path} --exclude refs/remotes/* --all))
puts "==> #{repo_bundle_path} generated in #{Time.now - start} seconds...\n"
end
end
def repos_path
@repos_path ||= GitalySetup.repos_path
end
def backup_path
Gitlab.config.backup.path
end
def pages_path
Gitlab.config.pages.path
end
def artifacts_path
Gitlab.config.artifacts.storage_path
end
def lfs_path
Gitlab.config.lfs.storage_path
end
def terraform_state_path
Gitlab.config.terraform_state.storage_path
end
def packages_path
Gitlab.config.packages.storage_path
end
# When no cached assets exist, manually hit the root path to create them
#
# Otherwise they'd be created by the first test, often timing out and
# causing a transient test failure
def eager_load_driver_server
return unless defined?(Capybara)
puts "Starting the Capybara driver server..."
Capybara.current_session.visit '/'
end
def factory_repo_path
@factory_repo_path ||= Rails.root.join('tmp', 'tests', factory_repo_name)
end
def forked_repo_path
@forked_repo_path ||= Rails.root.join('tmp', 'tests', forked_repo_name)
end
def factory_repo_bundle_path
"#{factory_repo_path}.bundle"
end
def forked_repo_bundle_path
"#{forked_repo_path}.bundle"
end
def seed_db
Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types
Gitlab::DatabaseImporters::WorkItems::HierarchyRestrictionsImporter.upsert_restrictions
end
private
# These are directories that should be preserved at cleanup time
def test_dirs
@test_dirs ||= %w[
frontend
gitaly
gitlab-shell
gitlab-test
gitlab-test.bundle
gitlab-test-fork
gitlab-test-fork.bundle
gitlab-workhorse
gitlab_workhorse_secret
]
end
def factory_repo_name
'gitlab-test'
end
def forked_repo_name
'gitlab-test-fork'
end
# Prevent developer git configurations from being persisted to test
# repositories
def git_env
{ 'GIT_TEMPLATE_DIR' => '' }
end
def set_repo_refs(repo_path, branch_sha)
IO.popen(%W[#{Gitlab.config.git.bin_path} -C #{repo_path} update-ref --stdin -z], "w") do |io|
branch_sha.each do |branch, sha|
io.write("update refs/heads/#{branch}\x00#{sha}\x00\x00")
end
end
$?.success?
end
def component_timed_setup(component, install_dir:, version:, task:, fresh_install: true, task_args: [])
start = Time.now
ensure_component_dir_name_is_correct!(component, install_dir)
# On CI, once installed, components never need update
return if File.exist?(install_dir) && ci?
if component_needs_update?(install_dir, version)
puts "==> Starting #{component} (#{version}) set up...\n"
# Cleanup the component entirely to ensure we start fresh
FileUtils.rm_rf(install_dir) if fresh_install
if ENV['SKIP_RAILS_ENV_IN_RAKE']
# When we run `scripts/setup-test-env`, we take care of loading the necessary dependencies
# so we can run the rake task programmatically.
Rake::Task[task].invoke(*task_args)
else
# In other cases, we run the task via `rake` so that the environment
# and dependencies are automatically loaded.
raise ComponentFailedToInstallError unless system('rake', "#{task}[#{task_args.join(',')}]")
end
yield if block_given?
puts "==> #{component} set up in #{Time.now - start} seconds...\n"
end
rescue ComponentFailedToInstallError
puts "\n#{component} failed to install, cleaning up #{install_dir}!\n"
FileUtils.rm_rf(install_dir)
exit 1
end
def ci?
ENV['CI'].present?
end
def ensure_component_dir_name_is_correct!(component, path)
actual_component_dir_name = File.basename(path)
expected_component_dir_name = component.parameterize
unless actual_component_dir_name == expected_component_dir_name
puts " #{component} install dir should be named '#{expected_component_dir_name}', not '#{actual_component_dir_name}' (full install path given was '#{path}')!\n"
exit 1
end
end
def component_needs_update?(component_folder, expected_version)
# Allow local overrides of the component for tests during development
return false if Rails.env.test? && File.symlink?(component_folder)
return false if component_matches_git_sha?(component_folder, expected_version)
return false if component_ahead_of_target?(component_folder, expected_version)
version = File.read(File.join(component_folder, 'VERSION')).strip
# Notice that this will always yield true when using branch versions
# (`=branch_name`), but that actually makes sure the server is always based
# on the latest branch revision.
version != expected_version
rescue Errno::ENOENT
true
end
def component_ahead_of_target?(component_folder, expected_version)
# The HEAD of the component_folder will be used as heuristic for the version
# of the binaries, allowing to use Git to determine if HEAD is later than
# the expected version. Note: Git considers HEAD to be an anchestor of HEAD.
_out, exit_status = Gitlab::Popen.popen(
%W[
#{Gitlab.config.git.bin_path}
-C #{component_folder}
merge-base --is-ancestor
#{expected_version} HEAD
]
)
exit_status == 0
end
def component_matches_git_sha?(component_folder, expected_version)
# Not a git SHA, so return early
return false unless expected_version =~ ::Gitlab::Git::COMMIT_ID
return false unless Dir.exist?(component_folder)
sha, exit_status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} rev-parse HEAD), component_folder)
return false if exit_status != 0
expected_version == sha.chomp
end
end
require_relative('../../../ee/spec/support/helpers/ee/test_env') if Gitlab.ee?
::TestEnv.prepend_mod_with('TestEnv')
::TestEnv.extend_mod_with('TestEnv')
|