Merge commit '4b92efd90cedaa0aff218d11fdce279701128bea' into object-storage-ee-to-ce-backport

7 files changed, 72 insertions, 103 deletions

diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 35ba729c156..247d7be7d78 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -23,6 +23,7 @@ namespace :gettext do
   desc 'Lint all po files in `locale/'
   task lint: :environment do
     require 'simple_po_parser'
+    require 'gitlab/utils'
 
     FastGettext.silence_errors
     files = Dir.glob(Rails.root.join('locale/*/gitlab.po'))
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index dfade1f3885..903e84359cd 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -387,14 +387,8 @@ namespace :gitlab do
   namespace :repo do
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
-
-        namespace_dirs.each do |namespace_dir|
-          repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
-          repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-        end
-      end
+      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
+      Rake::Task["gitlab:git:fsck"].execute
     end
   end
 
@@ -461,35 +455,4 @@ namespace :gitlab do
       puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
     end
   end
-
-  def check_repo_integrity(repo_dir)
-    puts "\nChecking repo at #{repo_dir.color(:yellow)}"
-
-    git_fsck(repo_dir)
-    check_config_lock(repo_dir)
-    check_ref_locks(repo_dir)
-  end
-
-  def git_fsck(repo_dir)
-    puts "Running `git fsck`".color(:yellow)
-    system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
-  end
-
-  def check_config_lock(repo_dir)
-    config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-    config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-    puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-  end
-
-  def check_ref_locks(repo_dir)
-    lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-    if lock_files.present?
-      puts "Ref lock files exist:".color(:red)
-      lock_files.each do |lock_file|
-        puts "  #{lock_file}"
-      end
-    else
-      puts "No ref lock files exist".color(:green)
-    end
-  end
 end
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index cf82134d97e..3f5dd2ae3b3 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -30,6 +30,20 @@ namespace :gitlab do
       end
     end
 
+    desc 'GitLab | Git | Check all repos integrity'
+    task fsck: :environment do
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
+        check_config_lock(repo)
+        check_ref_locks(repo)
+      end
+
+      if failures.empty?
+        puts "Done".color(:green)
+      else
+        output_failures(failures)
+      end
+    end
+
     def perform_git_cmd(cmd, message)
       puts "Starting #{message} on all repositories"
 
@@ -40,6 +54,8 @@ namespace :gitlab do
         else
           failures << repo
         end
+
+        yield(repo) if block_given?
       end
 
       failures
@@ -49,5 +65,24 @@ namespace :gitlab do
       puts "The following repositories reported errors:".color(:red)
       failures.each { |f| puts "- #{f}" }
     end
+
+    def check_config_lock(repo_dir)
+      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
+      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
+
+      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
+    end
+
+    def check_ref_locks(repo_dir)
+      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
+
+      if lock_files.present?
+        puts "Ref lock files exist:".color(:red)
+
+        lock_files.each { |lock_file| puts "  #{lock_file}" }
+      else
+        puts "No ref lock files exist".color(:green)
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 4d880c05f99..4507b841964 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -21,8 +21,8 @@ namespace :gitlab do
 
       command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
 
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
       Dir.chdir(args.dir) do
-        create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
           Bundler.with_original_env { run_command!(command) }
@@ -39,60 +39,7 @@ namespace :gitlab do
 
       # Exclude gitaly-ruby configuration because that depends on the gitaly
       # installation directory.
-      puts gitaly_configuration_toml(gitaly_ruby: false)
-    end
-
-    private
-
-    # We cannot create config.toml files for all possible Gitaly configuations.
-    # For instance, if Gitaly is running on another machine then it makes no
-    # sense to write a config.toml file on the current machine. This method will
-    # only generate a configuration for the most common and simplest case: when
-    # we have exactly one Gitaly process and we are sure it is running locally
-    # because it uses a Unix socket.
-    # For development and testing purposes, an extra storage is added to gitaly,
-    # which is not known to Rails, but must be explicitly stubbed.
-    def gitaly_configuration_toml(gitaly_ruby: true)
-      storages = []
-      address = nil
-
-      Gitlab.config.repositories.storages.each do |key, val|
-        if address
-          if address != val['gitaly_address']
-            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
-          end
-        elsif URI(val['gitaly_address']).scheme != 'unix'
-          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
-        else
-          address = val['gitaly_address']
-        end
-
-        storages << { name: key, path: val['path'] }
-      end
-
-      if Rails.env.test?
-        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
-      end
-
-      config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
-      config[:auth] = { token: 'secret' } if Rails.env.test?
-      config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
-      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
-      config[:bin_dir] = Gitlab.config.gitaly.client_path
-
-      TOML.dump(config)
-    end
-
-    def create_gitaly_configuration
-      File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
-        f.puts gitaly_configuration_toml
-      end
-    rescue Errno::EEXIST
-      puts "Skipping config.toml generation:"
-      puts "A configuration file already exists."
-    rescue ArgumentError => e
-      puts "Skipping config.toml generation:"
-      puts e.message
+      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
     end
   end
 end
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index 8a63f486fa3..c1182af1014 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -1,10 +1,13 @@
 require 'rainbow/ext/string'
+require 'gitlab/utils/strong_memoize'
 
 module Gitlab
   TaskFailedError = Class.new(StandardError)
   TaskAbortedByUserError = Class.new(StandardError)
 
   module TaskHelpers
+    include Gitlab::Utils::StrongMemoize
+
     extend self
 
     # Ask if the user wants to continue
@@ -105,16 +108,16 @@ module Gitlab
     end
 
     def gitlab_user?
-      return @is_gitlab_user unless @is_gitlab_user.nil?
-
-      current_user = run_command(%w(whoami)).chomp
-      @is_gitlab_user = current_user == gitlab_user
+      strong_memoize(:is_gitlab_user) do
+        current_user = run_command(%w(whoami)).chomp
+        current_user == gitlab_user
+      end
     end
 
     def warn_user_is_not_gitlab
-      return if @warned_user_not_gitlab
+      return if gitlab_user?
 
-      unless gitlab_user?
+      strong_memoize(:warned_user_not_gitlab) do
         current_user = run_command(%w(whoami)).chomp
 
         puts " Warning ".color(:black).background(:yellow)
@@ -122,14 +125,12 @@ module Gitlab
         puts "  Things may work\/fail for the wrong reasons."
         puts "  For correct results you should run this as user #{gitlab_user.color(:magenta)}."
         puts ""
-
-        @warned_user_not_gitlab = true
       end
     end
 
     def all_repos
       Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find|
           find.each_line do |path|
             yield path.chomp
           end
diff --git a/lib/tasks/gitlab/tcp_check.rake b/lib/tasks/gitlab/tcp_check.rake
new file mode 100644
index 00000000000..1400f57d6b9
--- /dev/null
+++ b/lib/tasks/gitlab/tcp_check.rake
@@ -0,0 +1,20 @@
+namespace :gitlab do
+  desc "GitLab | Check TCP connectivity to a specific host and port"
+  task :tcp_check, [:host, :port] => :environment do |_t, args|
+    unless args.host && args.port
+      puts "Please specify a host and port: `rake gitlab:tcp_check[example.com,80]`".color(:red)
+      exit 1
+    end
+
+    checker = Gitlab::TcpChecker.new(args.host, args.port)
+
+    if checker.check
+      puts "TCP connection from #{checker.local} to #{checker.remote} succeeded".color(:green)
+    else
+      puts "TCP connection to #{checker.remote} failed: #{checker.error}".color(:red)
+      puts
+      puts 'Check that host and port are correct, and that the traffic is permitted through any firewalls.'
+      exit 1
+    end
+  end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 9cc986535e1..c9e3eed82f2 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -6,6 +6,7 @@ require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_route
 require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
 require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
 require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
+require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
 
 desc 'GitLab | Sets up PostgreSQL'
 task setup_postgresql: :environment do
@@ -15,4 +16,5 @@ task setup_postgresql: :environment do
   IndexRoutesPathForLike.new.up
   AddLowerPathIndexToRedirectRoutes.new.up
   IndexRedirectRoutesPathForLike.new.up
+  AddIndexOnNamespacesLowerName.new.up
 end