summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/brakeman.rake2
-rw-r--r--lib/tasks/cache.rake7
-rw-r--r--lib/tasks/gemojione.rake3
-rw-r--r--lib/tasks/gettext.rake14
-rw-r--r--lib/tasks/gitlab/check.rake14
-rw-r--r--lib/tasks/gitlab/db.rake1
-rw-r--r--lib/tasks/gitlab/gitaly.rake59
-rw-r--r--lib/tasks/gitlab/info.rake3
-rw-r--r--lib/tasks/gitlab/shell.rake17
-rw-r--r--lib/tasks/gitlab/task_helpers.rb41
-rw-r--r--lib/tasks/gitlab/update_templates.rake8
-rw-r--r--lib/tasks/gitlab/workhorse.rake8
-rw-r--r--lib/tasks/import.rake142
-rw-r--r--lib/tasks/migrate/add_limits_mysql.rake2
-rw-r--r--lib/tasks/migrate/setup_postgresql.rake2
-rw-r--r--lib/tasks/spec.rake2
16 files changed, 267 insertions, 58 deletions
diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index 2301ec9b228..99b3168d9eb 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
task :brakeman do
# We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
# requests are welcome!
- if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
+ if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z))
puts 'Security check succeed'
else
puts 'Security check failed'
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index d55923673b1..125a3d560d6 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -21,12 +21,7 @@ namespace :cache do
end
end
- desc "GitLab | Clear database cache (in the background)"
- task db: :environment do
- ClearDatabaseCacheWorker.perform_async
- end
-
- task all: [:db, :redis]
+ task all: [:redis]
end
task clear: 'cache:clear:redis'
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 5293f5af12d..87ca39b079b 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -19,8 +19,9 @@ namespace :gemojione do
entry = {
category: emoji_hash['category'],
moji: emoji_hash['moji'],
+ description: emoji_hash['description'],
unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
- digest: hash_digest,
+ digest: hash_digest
}
resultant_emoji_map[name] = entry
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
new file mode 100644
index 00000000000..0aa21a4bd13
--- /dev/null
+++ b/lib/tasks/gettext.rake
@@ -0,0 +1,14 @@
+require "gettext_i18n_rails/tasks"
+
+namespace :gettext do
+ # Customize list of translatable files
+ # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
+ def files_to_translate
+ folders = %W(app lib config #{locale_path}).join(',')
+ exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',')
+
+ Dir.glob(
+ "{#{folders}}/**/*.{#{exts}}"
+ )
+ end
+end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index a9a48f7188f..f41c73154f5 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -431,8 +431,7 @@ namespace :gitlab do
def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
- gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
- puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
+ puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
@@ -443,15 +442,16 @@ namespace :gitlab do
break
end
- uid = uid_for(gitlab_shell_ssh_user)
- gid = gid_for(gitlab_shell_owner_group)
- if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+ user_id = uid_for(gitlab_shell_ssh_user)
+ root_group_id = gid_for('root')
+ group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
+ if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
puts "yes".color(:green)
else
puts "no".color(:red)
- puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+ puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
try_fixing_it(
- "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+ "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 5476438b8fa..139ab70e125 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -65,6 +65,7 @@ namespace :gitlab do
migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines
.map { |file| Rails.root.join(file.strip).to_s }
.select { |file| File.file?(file) }
+ .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) }
Gitlab::DowntimeCheck.new.check_and_print(migrations)
end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index c288e17ac8d..3c5bc0146a1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -1,23 +1,74 @@
namespace :gitlab do
namespace :gitaly do
desc "GitLab | Install or upgrade gitaly"
- task :install, [:dir] => :environment do |t, args|
+ task :install, [:dir, :repo] => :environment do |t, args|
+ require 'toml'
+
warn_user_is_not_gitlab
unless args.dir.present?
abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]")
end
+ args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
- tag = "v#{Gitlab::GitalyClient.expected_server_version}"
- repo = 'https://gitlab.com/gitlab-org/gitaly.git'
+ version = Gitlab::GitalyClient.expected_server_version
- checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+ checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
_, status = Gitlab::Popen.popen(%w[which gmake])
command = status.zero? ? 'gmake' : 'make'
Dir.chdir(args.dir) do
+ create_gitaly_configuration
run_command!([command])
end
end
+
+ desc "GitLab | Print storage configuration in TOML format"
+ task storage_config: :environment do
+ require 'toml'
+
+ puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
+ puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
+
+ config = Gitlab.config.repositories.storages.map do |key, val|
+ { name: key, path: val['path'] }
+ end
+
+ puts TOML.dump(storage: config)
+ end
+
+ private
+
+ # We cannot create config.toml files for all possible Gitaly configuations.
+ # For instance, if Gitaly is running on another machine then it makes no
+ # sense to write a config.toml file on the current machine. This method will
+ # only write a config.toml file in the most common and simplest case: the
+ # case where we have exactly one Gitaly process and we are sure it is
+ # running locally because it uses a Unix socket.
+ def create_gitaly_configuration
+ storages = []
+ address = nil
+
+ Gitlab.config.repositories.storages.each do |key, val|
+ if address
+ if address != val['gitaly_address']
+ raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
+ end
+ elsif URI(val['gitaly_address']).scheme != 'unix'
+ raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
+ else
+ address = val['gitaly_address']
+ end
+
+ storages << { name: key, path: val['path'] }
+ end
+
+ File.open("config.toml", "w") do |f|
+ f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages)
+ end
+ rescue ArgumentError => e
+ puts "Skipping config.toml generation:"
+ puts e.message
+ end
end
end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index a2a2db487b7..e3883278886 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -16,6 +16,8 @@ namespace :gitlab do
redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a
# check Git version
git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a
+ # check Go version
+ go_version = run_and_match(%w(go version), /go version (.+)/).to_a
puts ""
puts "System information".color(:yellow)
@@ -30,6 +32,7 @@ namespace :gitlab do
puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}"
puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}"
puts "Sidekiq Version:#{Sidekiq::VERSION}"
+ puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}"
# check database adapter
database_adapter = ActiveRecord::Base.connection.adapter_name.downcase
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index dd2fda54e62..ee2cdcdea1b 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -1,19 +1,18 @@
namespace :gitlab do
namespace :shell do
desc "GitLab | Install or upgrade gitlab-shell"
- task :install, [:tag, :repo] => :environment do |t, args|
+ task :install, [:repo] => :environment do |t, args|
warn_user_is_not_gitlab
default_version = Gitlab::Shell.version_required
- default_version_tag = "v#{default_version}"
- args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
+ args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
gitlab_url = Gitlab.config.gitlab.url
# gitlab-shell requires a / at the end of the url
gitlab_url += '/' unless gitlab_url.end_with?('/')
target_dir = Gitlab.config.gitlab_shell.path
- checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir)
+ checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
# Make sure we're on the right tag
Dir.chdir(target_dir) do
@@ -42,8 +41,14 @@ namespace :gitlab do
# Generate config.yml based on existing gitlab settings
File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
- # Launch installation process
- system(*%w(bin/install) + repository_storage_paths_args)
+ [
+ %w(bin/install) + repository_storage_paths_args,
+ %w(bin/compile)
+ ].each do |cmd|
+ unless Kernel.system(*cmd)
+ raise "command failed: #{cmd.join(' ')}"
+ end
+ end
end
# (Re)create hooks
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index cdba2262bc2..e3c9d3b491c 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -147,41 +147,30 @@ module Gitlab
Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
end
- def checkout_or_clone_tag(tag:, repo:, target_dir:)
- if Dir.exist?(target_dir)
- checkout_tag(tag, target_dir)
- else
- clone_repo(repo, target_dir)
- end
+ def checkout_or_clone_version(version:, repo:, target_dir:)
+ version =
+ if version.starts_with?("=")
+ version.sub(/\A=/, '') # tag or branch
+ else
+ "v#{version}" # tag
+ end
- reset_to_tag(tag, target_dir)
+ clone_repo(repo, target_dir) unless Dir.exist?(target_dir)
+ checkout_version(version, target_dir)
+ reset_to_version(version, target_dir)
end
def clone_repo(repo, target_dir)
run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
end
- def checkout_tag(tag, target_dir)
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet])
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}])
+ def checkout_version(version, target_dir)
+ run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet])
+ run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}])
end
- def reset_to_tag(tag_wanted, target_dir)
- tag =
- begin
- # First try to checkout without fetching
- # to avoid stalling tests if the Internet is down.
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}])
- rescue Gitlab::TaskFailedError
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin])
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}])
- end
-
- if tag
- run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}])
- else
- raise Gitlab::TaskFailedError
- end
+ def reset_to_version(version, target_dir)
+ run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}])
end
end
end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index dbdfb335a5c..59c32bbe7a4 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -5,7 +5,7 @@ namespace :gitlab do
end
def update(template)
- sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1]
+ sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
dir = File.join(vendor_directory, sub_dir)
unless clone_repository(template.repo_url, dir)
@@ -44,7 +44,11 @@ namespace :gitlab do
),
Template.new(
"https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
- /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/
+ /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
+ ),
+ Template.new(
+ "https://gitlab.com/gitlab-org/Dockerfile.git",
+ /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
)
].freeze
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index baea94bf8ca..e7ac0b5859f 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -1,16 +1,16 @@
namespace :gitlab do
namespace :workhorse do
desc "GitLab | Install or upgrade gitlab-workhorse"
- task :install, [:dir] => :environment do |t, args|
+ task :install, [:dir, :repo] => :environment do |t, args|
warn_user_is_not_gitlab
unless args.dir.present?
abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]")
end
+ args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git')
- tag = "v#{Gitlab::Workhorse.version}"
- repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git'
+ version = Gitlab::Workhorse.version
- checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+ checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
_, status = Gitlab::Popen.popen(%w[which gmake])
command = status.zero? ? 'gmake' : 'make'
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
new file mode 100644
index 00000000000..bc76d7edc55
--- /dev/null
+++ b/lib/tasks/import.rake
@@ -0,0 +1,142 @@
+require 'benchmark'
+require 'rainbow/ext/string'
+
+class GithubImport
+ def self.run!(*args)
+ new(*args).run!
+ end
+
+ def initialize(token, gitlab_username, project_path, extras)
+ @options = { url: 'https://api.github.com', token: token, verbose: true }
+ @project_path = project_path
+ @current_user = User.find_by_username(gitlab_username)
+ @github_repo = extras.empty? ? nil : extras.first
+ end
+
+ def run!
+ @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one!
+
+ raise 'No repo found!' unless @repo
+
+ show_warning!
+
+ @project = Project.find_by_full_path(@project_path) || new_project
+
+ import!
+ end
+
+ private
+
+ def show_warning!
+ puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
+ puts "Permission checks are ignored. Press any key to continue.".color(:red)
+
+ STDIN.getch
+
+ puts 'Starting the import (this could take a while)'.color(:green)
+ end
+
+ def import!
+ @project.import_start
+
+ timings = Benchmark.measure do
+ Github::Import.new(@project, @options).execute
+ end
+
+ puts "Import finished. Timings: #{timings}".color(:green)
+
+ @project.import_finish
+ end
+
+ def new_project
+ Project.transaction do
+ namespace_path, _sep, name = @project_path.rpartition('/')
+ namespace = find_or_create_namespace(namespace_path)
+
+ Projects::CreateService.new(
+ @current_user,
+ name: name,
+ path: name,
+ description: @repo['description'],
+ namespace_id: namespace.id,
+ visibility_level: visibility_level,
+ import_type: 'github',
+ import_source: @repo['full_name'],
+ skip_wiki: @repo['has_wiki']
+ ).execute
+ end
+ end
+
+ def find_or_create_namespace(names)
+ return @current_user.namespace if names == @current_user.namespace_path
+ return @current_user.namespace unless @current_user.can_create_group?
+
+ full_path_namespace = Namespace.find_by_full_path(names)
+
+ return full_path_namespace if full_path_namespace
+
+ names.split('/').inject(nil) do |parent, name|
+ begin
+ namespace = Group.create!(name: name,
+ path: name,
+ owner: @current_user,
+ parent: parent)
+ namespace.add_owner(@current_user)
+
+ namespace
+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+ Namespace.where(parent: parent).find_by_path_or_name(name)
+ end
+ end
+ end
+
+ def full_path_namespace(names)
+ @full_path_namespace ||= Namespace.find_by_full_path(names)
+ end
+
+ def visibility_level
+ @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+ end
+end
+
+class GithubRepos
+ def initialize(options, current_user, github_repo)
+ @options = options
+ @current_user = current_user
+ @github_repo = github_repo
+ end
+
+ def choose_one!
+ return found_github_repo if @github_repo
+
+ repos.each do |repo|
+ print "ID: #{repo['id'].to_s.bright}".color(:green)
+ print "\tName: #{repo['full_name']}\n".color(:green)
+ end
+
+ print 'ID? '.bright
+
+ repos.find { |repo| repo['id'] == repo_id }
+ end
+
+ def found_github_repo
+ repos.find { |repo| repo['full_name'] == @github_repo }
+ end
+
+ def repo_id
+ @repo_id ||= STDIN.gets.chomp.to_i
+ end
+
+ def repos
+ Github::Repositories.new(@options).fetch
+ end
+end
+
+namespace :import do
+ desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
+ task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
+ abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
+
+ GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras)
+ end
+end
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index 6ded519aff2..761f275d42a 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -1,7 +1,9 @@
require Rails.root.join('db/migrate/limits_to_mysql')
+require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
desc "GitLab | Add limits to strings in mysql database"
task add_limits_mysql: :environment do
puts "Adding limits to schema.rb for mysql"
LimitsToMysql.new.up
+ MarkdownCacheLimitsToMysql.new.up
end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 8938bc515f5..4108cee08b4 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe
require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
+require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
desc 'GitLab | Sets up PostgreSQL'
task setup_postgresql: :environment do
@@ -11,4 +12,5 @@ task setup_postgresql: :environment do
AddUsersLowerUsernameEmailIndexes.new.up
AddLowerPathIndexToRoutes.new.up
IndexRoutesPathForLike.new.up
+ IndexRedirectRoutesPathForLike.new.up
end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 602c60be828..2eddcb3c777 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -60,7 +60,7 @@ desc "GitLab | Run specs"
task :spec do
cmds = [
%w(rake gitlab:setup),
- %w(rspec spec),
+ %w(rspec spec)
]
run_commands(cmds)
end
View Lorry Controller Status