diff options
author | Nick Thomas <nick@gitlab.com> | 2019-01-09 01:58:44 +0000 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2019-01-09 01:58:44 +0000 |
commit | 74dca275d2f318bddd70c7bd9cecc2c1f5acac73 (patch) | |
tree | aae5831e80ec3599dbc846b91841aad431059aaa | |
parent | b05c06a909caad710f0203b81642916cc45cfc1f (diff) | |
parent | e4875cd2410018f639000118df7b6c2f1d7dd057 (diff) | |
download | gitlab-ce-74dca275d2f318bddd70c7bd9cecc2c1f5acac73.tar.gz |
Merge branch 'elastic-test-repos-ce' into 'master'
Backport of elastic-test-repos from EE
See merge request gitlab-org/gitlab-ce!24200
-rw-r--r-- | db/fixtures/development/04_project.rb | 200 | ||||
-rw-r--r-- | doc/development/rake_tasks.md | 8 | ||||
-rw-r--r-- | lib/gitlab/seeder.rb | 13 |
3 files changed, 142 insertions, 79 deletions
diff --git a/db/fixtures/development/04_project.rb b/db/fixtures/development/04_project.rb index aa8686ac7d8..9a5f7cf8175 100644 --- a/db/fixtures/development/04_project.rb +++ b/db/fixtures/development/04_project.rb @@ -1,94 +1,136 @@ require './spec/support/sidekiq' +# rubocop:disable Rails/Output + Sidekiq::Testing.inline! do Gitlab::Seeder.quiet do - project_urls = [ - 'https://gitlab.com/gitlab-org/gitlab-test.git', - 'https://gitlab.com/gitlab-org/gitlab-shell.git', - 'https://gitlab.com/gnuwget/wget2.git', - 'https://gitlab.com/Commit451/LabCoat.git', - 'https://github.com/jashkenas/underscore.git', - 'https://github.com/flightjs/flight.git', - 'https://github.com/twitter/typeahead.js.git', - 'https://github.com/h5bp/html5-boilerplate.git', - 'https://github.com/google/material-design-lite.git', - 'https://github.com/jlevy/the-art-of-command-line.git', - 'https://github.com/FreeCodeCamp/freecodecamp.git', - 'https://github.com/google/deepdream.git', - 'https://github.com/jtleek/datasharing.git', - 'https://github.com/WebAssembly/design.git', - 'https://github.com/airbnb/javascript.git', - 'https://github.com/tessalt/echo-chamber-js.git', - 'https://github.com/atom/atom.git', - 'https://github.com/mattermost/mattermost-server.git', - 'https://github.com/purifycss/purifycss.git', - 'https://github.com/facebook/nuclide.git', - 'https://github.com/wbkd/awesome-d3.git', - 'https://github.com/kilimchoi/engineering-blogs.git', - 'https://github.com/gilbarbara/logos.git', - 'https://github.com/reduxjs/redux.git', - 'https://github.com/awslabs/s2n.git', - 'https://github.com/arkency/reactjs_koans.git', - 'https://github.com/twbs/bootstrap.git', - 'https://github.com/chjj/ttystudio.git', - 'https://github.com/MostlyAdequate/mostly-adequate-guide.git', - 'https://github.com/octocat/Spoon-Knife.git', - 'https://github.com/opencontainers/runc.git', - 'https://github.com/googlesamples/android-topeka.git' - ] - - # You can specify how many projects you need during seed execution - size = ENV['SIZE'].present? ? ENV['SIZE'].to_i : 8 - - project_urls.first(size).each_with_index do |url, i| - group_path, project_path = url.split('/')[-2..-1] - - group = Group.find_by(path: group_path) - - unless group - group = Group.new( - name: group_path.titleize, - path: group_path - ) - group.description = FFaker::Lorem.sentence - group.save - - group.add_owner(User.first) - end + Gitlab::Seeder.without_gitaly_timeout do + project_urls = %w[ + https://gitlab.com/gitlab-org/gitlab-test.git + https://gitlab.com/gitlab-org/gitlab-shell.git + https://gitlab.com/gnuwget/wget2.git + https://gitlab.com/Commit451/LabCoat.git + https://github.com/jashkenas/underscore.git + https://github.com/flightjs/flight.git + https://github.com/twitter/typeahead.js.git + https://github.com/h5bp/html5-boilerplate.git + https://github.com/google/material-design-lite.git + https://github.com/jlevy/the-art-of-command-line.git + https://github.com/FreeCodeCamp/freecodecamp.git + https://github.com/google/deepdream.git + https://github.com/jtleek/datasharing.git + https://github.com/WebAssembly/design.git + https://github.com/airbnb/javascript.git + https://github.com/tessalt/echo-chamber-js.git + https://github.com/atom/atom.git + https://github.com/mattermost/mattermost-server.git + https://github.com/purifycss/purifycss.git + https://github.com/facebook/nuclide.git + https://github.com/wbkd/awesome-d3.git + https://github.com/kilimchoi/engineering-blogs.git + https://github.com/gilbarbara/logos.git + https://github.com/reduxjs/redux.git + https://github.com/awslabs/s2n.git + https://github.com/arkency/reactjs_koans.git + https://github.com/twbs/bootstrap.git + https://github.com/chjj/ttystudio.git + https://github.com/MostlyAdequate/mostly-adequate-guide.git + https://github.com/octocat/Spoon-Knife.git + https://github.com/opencontainers/runc.git + https://github.com/googlesamples/android-topeka.git + ] - project_path.gsub!(".git", "") + large_project_urls = %w[ + https://github.com/torvalds/linux.git + https://gitlab.gnome.org/GNOME/gimp.git + https://gitlab.gnome.org/GNOME/gnome-mud.git + https://gitlab.com/fdroid/fdroidclient.git + https://gitlab.com/inkscape/inkscape.git + https://github.com/gnachman/iTerm2.git + ] - params = { - import_url: url, - namespace_id: group.id, - name: project_path.titleize, - description: FFaker::Lorem.sentence, - visibility_level: Gitlab::VisibilityLevel.values.sample, - skip_disk_validation: true - } + def create_project(url, force_latest_storage: false) + group_path, project_path = url.split('/')[-2..-1] - if i % 2 == 0 - params[:storage_version] = Project::LATEST_STORAGE_VERSION - end + group = Group.find_by(path: group_path) + + unless group + group = Group.new( + name: group_path.titleize, + path: group_path + ) + group.description = FFaker::Lorem.sentence + group.save + + group.add_owner(User.first) + end + + project_path.gsub!(".git", "") + + params = { + import_url: url, + namespace_id: group.id, + name: project_path.titleize, + description: FFaker::Lorem.sentence, + visibility_level: Gitlab::VisibilityLevel.values.sample, + skip_disk_validation: true + } + + if force_latest_storage + params[:storage_version] = Project::LATEST_STORAGE_VERSION + end + + project = nil - project = nil + Sidekiq::Worker.skipping_transaction_check do + project = Projects::CreateService.new(User.first, params).execute - Sidekiq::Worker.skipping_transaction_check do - project = Projects::CreateService.new(User.first, params).execute + # Seed-Fu runs this entire fixture in a transaction, so the `after_commit` + # hook won't run until after the fixture is loaded. That is too late + # since the Sidekiq::Testing block has already exited. Force clearing + # the `after_commit` queue to ensure the job is run now. + project.send(:_run_after_commit_queue) + project.import_state.send(:_run_after_commit_queue) + end - # Seed-Fu runs this entire fixture in a transaction, so the `after_commit` - # hook won't run until after the fixture is loaded. That is too late - # since the Sidekiq::Testing block has already exited. Force clearing - # the `after_commit` queue to ensure the job is run now. - project.send(:_run_after_commit_queue) - project.import_state.send(:_run_after_commit_queue) + if project.valid? && project.valid_repo? + print '.' + else + puts project.errors.full_messages + print 'F' + end end - if project.valid? && project.valid_repo? - print '.' - else - puts project.errors.full_messages - print 'F' + # You can specify how many projects you need during seed execution + size = ENV['SIZE'].present? ? ENV['SIZE'].to_i : 8 + + project_urls.first(size).each_with_index do |url, i| + create_project(url, force_latest_storage: i.even?) + end + + if ENV['LARGE_PROJECTS'].present? + large_project_urls.each(&method(:create_project)) + + if ENV['FORK'].present? + puts "\nGenerating forks" + + project_name = ENV['FORK'] == 'true' ? 'torvalds/linux' : ENV['FORK'] + + project = Project.find_by_full_path(project_name) + + User.offset(1).first(5).each do |user| + new_project = Projects::ForkService.new(project, user).execute + + if new_project.valid? && (new_project.valid_repo? || new_project.import_state.scheduled?) + print '.' + else + new_project.errors.full_messages.each do |error| + puts "#{new_project.full_path}: #{error}" + end + print 'F' + end + end + end end end end diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 2ad748d4802..ae9bf863419 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -38,6 +38,14 @@ Note that since you can't see the questions from stdout, you might just want to `echo 'yes'` to keep it running. It would still print the errors on stderr so no worries about missing errors. +### Extra Project seed options + +There are a few environment flags you can pass to change how projects are seeded + +- `SIZE`: defaults to `8`, max: `32`. Amount of projects to create. +- `LARGE_PROJECTS`: defaults to false. If set will clone 6 large projects to help with testing. +- `FORK`: defaults to false. If set to `true` will fork `torvalds/linux` five times. Can also be set to an existing project full_path and it will fork that instead. + ### Notes for MySQL Since the seeds would contain various UTF-8 characters, such as emojis or so, diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 84a51773276..8e2f16271eb 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -26,6 +26,19 @@ module Gitlab puts "\nOK".color(:green) end + def self.without_gitaly_timeout + # Remove Gitaly timeout + old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0) + # Otherwise we still see the default value when running seed_fu + ApplicationSetting.expire + + yield + ensure + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout) + ApplicationSetting.expire + end + def self.mute_notifications NotificationService.prepend(MuteNotifications) end |