Merge branch 'master' into new-resolvable-discussion

15 files changed, 311 insertions, 39 deletions

diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 56c597dffcb..70d0d57204d 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -142,7 +142,7 @@ module API
         project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, ''))
 
         begin
-          Gitlab::GitalyClient::Notifications.new(project.repository_storage, relative_path).post_receive
+          Gitlab::GitalyClient::Notifications.new(project.repository).post_receive
         rescue GRPC::Unavailable => e
           render_api_error(e, 500)
         end
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index 35ea2e0ef59..b07c68d1498 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -5,7 +5,11 @@ require 'gitlab/email/handler/unsubscribe_handler'
 module Gitlab
   module Email
     module Handler
-      HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze
+      HANDLERS = [
+        UnsubscribeHandler,
+        CreateNoteHandler,
+        CreateIssueHandler
+      ].freeze
 
       def self.for(mail, mail_key)
         HANDLERS.find do |klass|
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 9e338282e96..fc473b2c21e 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -968,6 +968,14 @@ module Gitlab
         @attributes.attributes(path)
       end
 
+      def gitaly_repository
+        Gitlab::GitalyClient::Util.repository(@repository_storage, @relative_path)
+      end
+
+      def gitaly_channel
+        Gitlab::GitalyClient.get_channel(@repository_storage)
+      end
+
       private
 
       # Get the content of a blob for a given commit.  If the blob is a commit
@@ -1247,7 +1255,7 @@ module Gitlab
       end
 
       def gitaly_ref_client
-        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(@repository_storage, @relative_path)
+        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
index f15faebe27e..b7f39f3ef0b 100644
--- a/lib/gitlab/gitaly_client/commit.rb
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -7,14 +7,13 @@ module Gitlab
 
       class << self
         def diff_from_parent(commit, options = {})
-          project   = commit.project
-          channel   = GitalyClient.get_channel(project.repository_storage)
-          stub      = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel)
-          repo      = Gitaly::Repository.new(path: project.repository.path_to_repo)
-          parent    = commit.parents[0]
+          repository = commit.project.repository
+          gitaly_repo = repository.gitaly_repository
+          stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
+          parent = commit.parents[0]
           parent_id = parent ? parent.id : EMPTY_TREE_ID
-          request   = Gitaly::CommitDiffRequest.new(
-            repository: repo,
+          request = Gitaly::CommitDiffRequest.new(
+            repository: gitaly_repo,
             left_commit_id: parent_id,
             right_commit_id: commit.id
           )
@@ -23,12 +22,10 @@ module Gitlab
         end
 
         def is_ancestor(repository, ancestor_id, child_id)
-          project = Project.find_by_path(repository.path)
-          channel = GitalyClient.get_channel(project.repository_storage)
-          stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: channel)
-          repo = Gitaly::Repository.new(path: repository.path_to_repo)
+          gitaly_repo = repository.gitaly_repository
+          stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
           request = Gitaly::CommitIsAncestorRequest.new(
-            repository: repo,
+            repository: gitaly_repo,
             ancestor_id: ancestor_id,
             child_id: child_id
           )
diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb
index f0d93ded91b..a94a54883db 100644
--- a/lib/gitlab/gitaly_client/notifications.rb
+++ b/lib/gitlab/gitaly_client/notifications.rb
@@ -3,13 +3,14 @@ module Gitlab
     class Notifications
       attr_accessor :stub
 
-      def initialize(repository_storage, relative_path)
-        @channel, @repository = Util.process_path(repository_storage, relative_path)
-        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: @channel)
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
       end
 
       def post_receive
-        request = Gitaly::PostReceiveRequest.new(repository: @repository)
+        request = Gitaly::PostReceiveRequest.new(repository: @gitaly_repo)
         @stub.post_receive(request)
       end
     end
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index fcdf452d567..d3c0743db4e 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -3,23 +3,24 @@ module Gitlab
     class Ref
       attr_accessor :stub
 
-      def initialize(repository_storage, relative_path)
-        @channel, @repository = Util.process_path(repository_storage, relative_path)
-        @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: @channel)
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
       end
 
       def default_branch_name
-        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @repository)
+        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo)
         stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '')
       end
 
       def branch_names
-        request = Gitaly::FindAllBranchNamesRequest.new(repository: @repository)
+        request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
         consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/')
       end
 
       def tag_names
-        request = Gitaly::FindAllTagNamesRequest.new(repository: @repository)
+        request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
         consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/')
       end
 
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index d272c25d1f9..4acd297f5cb 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -1,12 +1,14 @@
 module Gitlab
   module GitalyClient
     module Util
-      def self.process_path(repository_storage, relative_path)
-        channel = GitalyClient.get_channel(repository_storage)
-        storage_path = Gitlab.config.repositories.storages[repository_storage]['path']
-        repository = Gitaly::Repository.new(path: File.join(storage_path, relative_path))
-
-        [channel, repository]
+      class << self
+        def repository(repository_storage, relative_path)
+          Gitaly::Repository.new(
+            path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path),
+            storage_name: repository_storage,
+            relative_path: relative_path,
+          )
+        end
       end
     end
   end
diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb
new file mode 100644
index 00000000000..7de6d4d9367
--- /dev/null
+++ b/lib/gitlab/health_checks/base_abstract_check.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module HealthChecks
+    module BaseAbstractCheck
+      def name
+        super.demodulize.underscore
+      end
+
+      def human_name
+        name.sub(/_check$/, '').capitalize
+      end
+
+      def readiness
+        raise NotImplementedError
+      end
+
+      def liveness
+        HealthChecks::Result.new(true)
+      end
+
+      def metrics
+        []
+      end
+
+      protected
+
+      def metric(name, value, **labels)
+        Metric.new(name, value, labels)
+      end
+
+      def with_timing(proc)
+        start = Time.now
+        result = proc.call
+        yield result, Time.now.to_f - start.to_f
+      end
+
+      def catch_timeout(seconds, &block)
+        begin
+          Timeout.timeout(seconds.to_i, &block)
+        rescue Timeout::Error => ex
+          ex
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb
new file mode 100644
index 00000000000..fd94984f8a2
--- /dev/null
+++ b/lib/gitlab/health_checks/db_check.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module HealthChecks
+    class DbCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'db_ping'
+        end
+
+        def is_successful?(result)
+          result == '1'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            if Gitlab::Database.postgresql?
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')
+            else
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
new file mode 100644
index 00000000000..df962d203b7
--- /dev/null
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  module HealthChecks
+    class FsShardsCheck
+      extend BaseAbstractCheck
+
+      class << self
+        def readiness
+          repository_storages.map do |storage_name|
+            begin
+              tmp_file_path = tmp_file_path(storage_name)
+
+              if !storage_stat_test(storage_name)
+                HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
+              elsif !storage_write_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
+              elsif !storage_read_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
+              else
+                HealthChecks::Result.new(true, nil, shard: storage_name)
+              end
+            rescue RuntimeError => ex
+              message = "unexpected error #{ex} when checking storage #{storage_name}"
+              Rails.logger.error(message)
+              HealthChecks::Result.new(false, message, shard: storage_name)
+            ensure
+              delete_test_file(tmp_file_path)
+            end
+          end
+        end
+
+        def metrics
+          repository_storages.flat_map do |storage_name|
+            tmp_file_path = tmp_file_path(storage_name)
+            [
+              operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name),
+              operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name),
+              operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
+            ].flatten
+          end
+        end
+
+        private
+
+        RANDOM_STRING = SecureRandom.hex(1000).freeze
+
+        def operation_metrics(ok_metric, latency_metric, operation, **labels)
+          with_timing operation do |result, elapsed|
+            [
+              metric(latency_metric, elapsed, **labels),
+              metric(ok_metric, result ? 1 : 0, **labels)
+            ]
+          end
+        rescue RuntimeError => ex
+          Rails.logger("unexpected error #{ex} when checking #{ok_metric}")
+          [metric(ok_metric, 0, **labels)]
+        end
+
+        def repository_storages
+          @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
+        end
+
+        def storages_paths
+          @storage_paths ||= Gitlab.config.repositories.storages
+        end
+
+        def with_timeout(args)
+          %w{timeout 1}.concat(args)
+        end
+
+        def tmp_file_path(storage_name)
+          Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path }
+        end
+
+        def path(storage_name)
+          storages_paths&.dig(storage_name, 'path')
+        end
+
+        def storage_stat_test(storage_name)
+          stat_path = File.join(path(storage_name), '.')
+          begin
+            _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} }))
+            status == 0
+          rescue Errno::ENOENT
+            File.exist?(stat_path) && File::Stat.new(stat_path).readable?
+          end
+        end
+
+        def storage_write_test(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT
+          written_bytes == RANDOM_STRING.length
+        end
+
+        def storage_read_test(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          file_contents = File.read(tmp_path) rescue Errno::ENOENT
+          file_contents == RANDOM_STRING
+        end
+
+        def delete_test_file(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} }))
+          status == 0
+        rescue Errno::ENOENT
+          File.delete(tmp_path) rescue Errno::ENOENT
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
new file mode 100644
index 00000000000..1a2eab0b005
--- /dev/null
+++ b/lib/gitlab/health_checks/metric.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Metric = Struct.new(:name, :value, :labels)
+end
diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb
new file mode 100644
index 00000000000..57bbe5b3ad0
--- /dev/null
+++ b/lib/gitlab/health_checks/redis_check.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module HealthChecks
+    class RedisCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'redis_ping'
+        end
+
+        def is_successful?(result)
+          result == 'PONG'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            Gitlab::Redis.with(&:ping)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
new file mode 100644
index 00000000000..8086760023e
--- /dev/null
+++ b/lib/gitlab/health_checks/result.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Result = Struct.new(:success, :message, :labels)
+end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
new file mode 100644
index 00000000000..fbe1645c1b1
--- /dev/null
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module HealthChecks
+    module SimpleAbstractCheck
+      include BaseAbstractCheck
+
+      def readiness
+        check_result = check
+        if is_successful?(check_result)
+          HealthChecks::Result.new(true)
+        elsif check_result.is_a?(Timeout::Error)
+          HealthChecks::Result.new(false, "#{human_name} check timed out")
+        else
+          HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}")
+        end
+      end
+
+      def metrics
+        with_timing method(:check) do |result, elapsed|
+          Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result)
+          [
+            metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0),
+            metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0),
+            metric("#{metric_prefix}_latency", elapsed)
+          ]
+        end
+      end
+
+      private
+
+      def metric_prefix
+        raise NotImplementedError
+      end
+
+      def is_successful?(result)
+        raise NotImplementedError
+      end
+
+      def check
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index a8a7bf9bc12..e6e40f6945d 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -24,14 +24,8 @@ module Gitlab
         }
 
         if Gitlab.config.gitaly.enabled
-          storage = repository.project.repository_storage
-          address = Gitlab::GitalyClient.get_address(storage)
-          # TODO: use GitalyClient code to assemble the Repository message
-          params[:Repository] = Gitaly::Repository.new(
-            path: repo_path,
-            storage_name: storage,
-            relative_path: Gitlab::RepoPath.strip_storage_path(repo_path),
-          ).to_h
+          address = Gitlab::GitalyClient.get_address(repository.project.repository_storage)
+          params[:Repository] = repository.gitaly_repository.to_h
 
           feature_enabled = case action.to_s
                             when 'git_receive_pack'