19 files changed, 158 insertions, 42 deletions

diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb
index eeab7791643..f323d2e0f7a 100644
--- a/lib/gitlab/auth/ldap/access.rb
+++ b/lib/gitlab/auth/ldap/access.rb
@@ -92,12 +92,12 @@ module Gitlab
           if provider
             Gitlab::AppLogger.info(
               "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
-              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+              "blocking GitLab user \"#{user.name}\" (#{user.email})"
             )
           else
             Gitlab::AppLogger.info(
               "Account is not provided by LDAP, " \
-              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+              "blocking GitLab user \"#{user.name}\" (#{user.email})"
             )
           end
         end
@@ -107,7 +107,7 @@ module Gitlab
 
           Gitlab::AppLogger.info(
             "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
-            "unblocking Gitlab user \"#{user.name}\" (#{user.email})"
+            "unblocking GitLab user \"#{user.name}\" (#{user.email})"
           )
         end
       end
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 3236abfa43f..1ffc2639237 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -30,8 +30,9 @@ module Gitlab
       note_events = event_counts(date_from, :merge_requests)
         .having(action: [Event::COMMENTED])
 
-      union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events])
-      events = Event.find_by_sql(union.to_sql).map(&:attributes)
+      events = Event
+        .from_union([repo_events, issue_events, mr_events, note_events])
+        .map(&:attributes)
 
       @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities|
         activities[event["date"]] += event["total_amount"]
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 8eacad078c8..42f9605f5ac 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -249,5 +249,21 @@ module Gitlab
     end
 
     private_class_method :database_version
+
+    def self.add_post_migrate_path_to_rails(force: false)
+      return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
+
+      Rails.application.config.paths['db'].each do |db_path|
+        path = Rails.root.join(db_path, 'post_migrate').to_s
+
+        unless Rails.application.config.paths['db/migrate'].include? path
+          Rails.application.config.paths['db/migrate'] << path
+
+          # Rails memoizes migrations at certain points where it won't read the above
+          # path just yet. As such we must also update the following list of paths.
+          ActiveRecord::Migrator.migrations_paths << path
+        end
+      end
+    end
   end
 end
diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb
index d32837f5793..7d334a79009 100644
--- a/lib/gitlab/database/grant.rb
+++ b/lib/gitlab/database/grant.rb
@@ -2,6 +2,8 @@ module Gitlab
   module Database
     # Model that can be used for querying permissions of a SQL user.
     class Grant < ActiveRecord::Base
+      include FromUnion
+
       self.table_name =
         if Database.postgresql?
           'information_schema.role_table_grants'
@@ -42,9 +44,7 @@ module Gitlab
               .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
           ]
 
-          union = SQL::Union.new(queries).to_sql
-
-          Grant.from("(#{union}) privs").any?
+          Grant.from_union(queries, alias_as: 'privs').any?
         end
       end
     end
diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb
new file mode 100644
index 00000000000..2a6f39c6a27
--- /dev/null
+++ b/lib/gitlab/database/subquery.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Subquery
+      class << self
+        def self_join(relation)
+          t = relation.arel_table
+          t2 = relation.arel.as('t2')
+
+          relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index d2ca7a070fa..fb117baca9e 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -20,8 +20,9 @@ module Gitlab
         DiffViewer::Image
       ].sort_by { |v| v.binary? ? 0 : 1 }.freeze
 
-      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil)
+      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil)
         @diff = diff
+        @stats = stats
         @repository = repository
         @diff_refs = diff_refs
         @fallback_diff_refs = fallback_diff_refs
@@ -165,11 +166,11 @@ module Gitlab
       end
 
       def added_lines
-        diff_lines.count(&:added?)
+        @stats&.additions || diff_lines.count(&:added?)
       end
 
       def removed_lines
-        diff_lines.count(&:removed?)
+        @stats&.deletions || diff_lines.count(&:removed?)
       end
 
       def file_identifier
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 2acb0e43b69..b79ff771a2b 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -2,23 +2,27 @@ module Gitlab
   module Diff
     module FileCollection
       class Base
+        include Gitlab::Utils::StrongMemoize
+
         attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable
 
         delegate :count, :size, :real_size, to: :diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true)
         end
 
         def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
           diff_options = self.class.default_options.merge(diff_options || {})
 
           @diffable = diffable
+          @include_stats = diff_options.delete(:include_stats)
           @diffs = diffable.raw_diffs(diff_options)
           @project = project
           @diff_options = diff_options
           @diff_refs = diff_refs
           @fallback_diff_refs = fallback_diff_refs
+          @repository = project.repository
         end
 
         def diff_files
@@ -43,10 +47,27 @@ module Gitlab
 
         private
 
+        def diff_stats_collection
+          strong_memoize(:diff_stats) do
+            # There are scenarios where we don't need to request Diff Stats,
+            # when caching for instance.
+            next unless @include_stats
+            next unless diff_refs
+
+            @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha)
+          end
+        end
+
         def decorate_diff!(diff)
           return diff if diff.is_a?(File)
 
-          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
+          stats = diff_stats_collection&.find_by_path(diff.new_path)
+
+          Gitlab::Diff::File.new(diff,
+                                 repository: project.repository,
+                                 diff_refs: diff_refs,
+                                 fallback_diff_refs: fallback_diff_refs,
+                                 stats: stats)
         end
       end
     end
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 978962ab2eb..4b6016e00bc 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -116,6 +116,10 @@ module Gitlab
         end
       end
 
+      def diff_options
+        { paths: paths, expanded: true, include_stats: false }
+      end
+
       def diff_line(repository)
         @diff_line ||= diff_file(repository)&.line_for_position(self)
       end
@@ -130,7 +134,7 @@ module Gitlab
         return unless diff_refs.complete?
         return unless comparison = diff_refs.compare_in(repository.project)
 
-        comparison.diffs(paths: paths, expanded: true).diff_files.first
+        comparison.diffs(diff_options).diff_files.first
       end
 
       def get_formatter_class(type)
diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb
new file mode 100644
index 00000000000..d4033f56387
--- /dev/null
+++ b/lib/gitlab/git/diff_stats_collection.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class DiffStatsCollection
+      include Gitlab::Utils::StrongMemoize
+      include Enumerable
+
+      def initialize(diff_stats)
+        @collection = diff_stats
+      end
+
+      def each(&block)
+        @collection.each(&block)
+      end
+
+      def find_by_path(path)
+        indexed_by_path[path]
+      end
+
+      private
+
+      def indexed_by_path
+        strong_memoize(:indexed_by_path) do
+          index_by { |stats| stats.path }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 1b8d320ff3b..3d5a63bdbac 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -438,6 +438,16 @@ module Gitlab
         Gitlab::Git::DiffCollection.new(iterator, options)
       end
 
+      def diff_stats(left_id, right_id)
+        stats = wrapped_gitaly_errors do
+          gitaly_commit_client.diff_stats(left_id, right_id)
+        end
+
+        Gitlab::Git::DiffStatsCollection.new(stats)
+      rescue CommandError, TypeError
+        Gitlab::Git::DiffStatsCollection.new([])
+      end
+
       # Returns a RefName for a given SHA
       def ref_name_for_sha(ref_path, sha)
         raise ArgumentError, "sha can't be empty" unless sha.present?
@@ -581,10 +591,6 @@ module Gitlab
         end
       end
 
-      def user_to_committer(user)
-        Gitlab::Git.committer_hash(email: user.email, name: user.name)
-      end
-
       # Delete the specified branch from the repository
       def delete_branch(branch_name)
         wrapped_gitaly_errors do
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 12307338972..302341d9541 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -174,10 +174,29 @@ module Gitlab
     end
     private_class_method :current_transaction_labels
 
+    # For some time related tasks we can't rely on `Time.now` since it will be
+    # affected by Timecop in some tests, and the clock of some gitaly-related
+    # components (grpc's c-core and gitaly server) use system time instead of
+    # timecop's time, so tests will fail.
+    # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent
+    # timecop.
+    def self.real_time
+      Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))
+    end
+    private_class_method :real_time
+
+    def self.authorization_token(storage)
+      token = token(storage).to_s
+      issued_at = real_time.to_i.to_s
+      hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at)
+
+      "v2.#{hmac}.#{issued_at}"
+    end
+    private_class_method :authorization_token
+
     def self.request_kwargs(storage, timeout, remote_storage: nil)
-      encoded_token = Base64.strict_encode64(token(storage).to_s)
       metadata = {
-        'authorization' => "Bearer #{encoded_token}",
+        'authorization' => "Bearer #{authorization_token(storage)}",
         'client_name' => CLIENT_NAME
       }
 
@@ -195,12 +214,7 @@ module Gitlab
 
       return result unless timeout > 0
 
-      # Do not use `Time.now` for deadline calculation, since it
-      # will be affected by Timecop in some tests, but grpc's c-core
-      # uses system time instead of timecop's time, so tests will fail
-      # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will
-      # circumvent timecop
-      deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout
+      deadline = real_time + timeout
       result[:deadline] = deadline
 
       result
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index f65d7383dc7..6c95abdcb4b 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -172,6 +172,17 @@ module Gitlab
         consume_commits_response(response)
       end
 
+      def diff_stats(left_commit_sha, right_commit_sha)
+        request = Gitaly::DiffStatsRequest.new(
+          repository: @gitaly_repo,
+          left_commit_id: left_commit_sha,
+          right_commit_id: right_commit_sha
+        )
+
+        response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout)
+        response.flat_map(&:stats)
+      end
+
       def find_all_commits(opts = {})
         request = Gitaly::FindAllCommitsRequest.new(
           repository: @gitaly_repo,
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 8e530de174d..26d1f53f26c 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -13,7 +13,7 @@ module Gitlab
         Storage is invalid because it has no `path` key.
 
         For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example.
-        If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`.
+        If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`.
       MSG
 
       # This class will give easily recognizable NoMethodErrors
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
index b74e7b10448..8fbfa1a86bf 100644
--- a/lib/gitlab/group_hierarchy.rb
+++ b/lib/gitlab/group_hierarchy.rb
@@ -89,14 +89,14 @@ module Gitlab
       ancestors_table = ancestors.alias_to(groups_table)
       descendants_table = descendants.alias_to(groups_table)
 
-      union = SQL::Union.new([model.unscoped.from(ancestors_table),
-                              model.unscoped.from(descendants_table)])
-
       relation = model
         .unscoped
         .with
         .recursive(ancestors.to_arel, descendants.to_arel)
-        .from("(#{union.to_sql}) #{model.table_name}")
+        .from_union([
+          model.unscoped.from(ancestors_table),
+          model.unscoped.from(descendants_table)
+        ])
 
       read_only(relation)
     end
diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb
index e3da1634fa5..448c3f3a7d8 100644
--- a/lib/gitlab/project_authorizations/with_nested_groups.rb
+++ b/lib/gitlab/project_authorizations/with_nested_groups.rb
@@ -49,13 +49,11 @@ module Gitlab
             .where('p_ns.share_with_group_lock IS FALSE')
         ]
 
-        union = Gitlab::SQL::Union.new(relations)
-
         ProjectAuthorization
           .unscoped
           .with
           .recursive(cte.to_arel)
-          .select_from_union(union)
+          .select_from_union(relations)
       end
 
       private
diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb
index 7d0c00c7f36..ed2287dcc7e 100644
--- a/lib/gitlab/project_authorizations/without_nested_groups.rb
+++ b/lib/gitlab/project_authorizations/without_nested_groups.rb
@@ -24,11 +24,9 @@ module Gitlab
           user.groups.joins(:shared_projects).select_for_project_authorization
         ]
 
-        union = Gitlab::SQL::Union.new(relations)
-
         ProjectAuthorization
           .unscoped
-          .select_from_union(union)
+          .select_from_union(relations)
       end
     end
   end
diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb
index 831da45191f..b08d9a99e99 100644
--- a/lib/gitlab/template/finders/global_template_finder.rb
+++ b/lib/gitlab/template/finders/global_template_finder.rb
@@ -1,4 +1,4 @@
-# Searches and reads file present on Gitlab installation directory
+# Searches and reads file present on GitLab installation directory
 module Gitlab
   module Template
     module Finders
diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb
index 29bc2393ff9..9140ace879f 100644
--- a/lib/gitlab/template/finders/repo_template_finder.rb
+++ b/lib/gitlab/template/finders/repo_template_finder.rb
@@ -1,4 +1,4 @@
-# Searches and reads files present on each Gitlab project repository
+# Searches and reads files present on each GitLab project repository
 module Gitlab
   module Template
     module Finders
diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb
index 09652a4fb5e..bd0d24e4369 100644
--- a/lib/gitlab/user_extractor.rb
+++ b/lib/gitlab/user_extractor.rb
@@ -18,7 +18,7 @@ module Gitlab
     def users
       return User.none unless @text.present?
 
-      @users ||= User.from("(#{union.to_sql}) users")
+      @users ||= User.from_union(union_relations)
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
@@ -43,13 +43,13 @@ module Gitlab
 
     private
 
-    def union
+    def union_relations
       relations = []
 
       relations << User.by_any_email(emails) if emails.any?
       relations << User.by_username(usernames) if usernames.any?
 
-      Gitlab::SQL::Union.new(relations)
+      relations
     end
   end
 end