Merge branch 'speedup-gitlab-assets-compile' into 'master'

Cache assets in gitlab:assets:compile

Closes #56691

See merge request gitlab-org/gitlab-ce!24542

9 files changed, 72 insertions, 16 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ee9eaeae723..b93a79de994 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -386,20 +386,25 @@ flaky-examples-check:
     - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json
     - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT
 
+.assets-compile-cache: &assets-compile-cache
+  cache:
+    key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v3"
+    paths:
+      - vendor/ruby/
+      - .yarn-cache/
+      - tmp/cache/assets/sprockets
+
 compile-assets:
   <<: *dedicated-runner
   <<: *except-docs
   <<: *use-pg
   stage: prepare
-  cache:
-    <<: *default-cache
   script:
     - node --version
-    - date
     - yarn install --frozen-lockfile --cache-folder .yarn-cache
-    - date
     - free -m
     - bundle exec rake gitlab:assets:compile
+    - scripts/clean-old-cached-assets
   variables:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
@@ -408,6 +413,7 @@ compile-assets:
     paths:
       - node_modules
       - public/assets
+  <<: *assets-compile-cache
 
 setup-test-env:
   <<: *dedicated-runner
@@ -628,7 +634,9 @@ gitlab:setup-mysql:
 gitlab:assets:compile:
   <<: *dedicated-no-docs-pull-cache-job
   image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1
-  dependencies: []
+  dependencies:
+    - setup-test-env
+    - compile-assets
   services:
     - docker:stable-dind
   variables:
@@ -642,18 +650,19 @@ gitlab:assets:compile:
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
   script:
-    - date
+    - node --version
     - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
-    - date
     - free -m
     - bundle exec rake gitlab:assets:compile
-    - scripts/build_assets_image
+    - time scripts/build_assets_image
+    - scripts/clean-old-cached-assets
   artifacts:
     name: webpack-report
     expire_in: 31d
     paths:
       - webpack-report/
       - public/assets/
+  <<: *assets-compile-cache
   only:
     - //@gitlab-org/gitlab-ce
     - //@gitlab-org/gitlab-ee
diff --git a/config/initializers/sprockets_base_file_digest_key.rb b/config/initializers/sprockets_base_file_digest_key.rb
new file mode 100644
index 00000000000..81ff3812091
--- /dev/null
+++ b/config/initializers/sprockets_base_file_digest_key.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Sprockets::Base.prepend(Gitlab::Patch::SprocketsBaseFileDigestKey)
diff --git a/lib/gitlab/patch/sprockets_base_file_digest_key.rb b/lib/gitlab/patch/sprockets_base_file_digest_key.rb
new file mode 100644
index 00000000000..3925cdbbada
--- /dev/null
+++ b/lib/gitlab/patch/sprockets_base_file_digest_key.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# This monkey patch prevent cache ballooning when caching tmp/cache/assets/sprockets
+# on the CI. See https://github.com/rails/sprockets/issues/563 and
+# https://github.com/rails/sprockets/compare/3.x...jmreid:no-mtime-for-digest-key.
+module Gitlab
+  module Patch
+    module SprocketsBaseFileDigestKey
+      def file_digest(path)
+        if stat = self.stat(path)
+          digest = self.stat_digest(path, stat)
+          integrity_uri = self.hexdigest_integrity_uri(digest)
+
+          key = Sprockets::UnloadedAsset.new(path, self).file_digest_key(integrity_uri)
+          cache.fetch(key) do
+            digest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 224bb648d8f..8532845f3cb 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'rainbow/ext/string'
-require 'gitlab/utils/strong_memoize'
+require_dependency 'gitlab/utils/strong_memoize'
 
 # rubocop:disable Rails/Output
 module Gitlab
@@ -13,6 +13,12 @@ module Gitlab
 
     extend self
 
+    def invoke_and_time_task(task)
+      start = Time.now
+      Rake::Task[task].invoke
+      puts "`#{task}` finished in #{Time.now - start} seconds"
+    end
+
     # Ask if the user wants to continue
     #
     # Returns "yes" the user chose to continue
diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb
index fc237861e2f..48ba13b8561 100644
--- a/lib/gitlab/utils/merge_hash.rb
+++ b/lib/gitlab/utils/merge_hash.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_dependency 'gitlab/utils'
+
 module Gitlab
   module Utils
     module MergeHash
diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb
index c87e97d0213..f5299439fce 100644
--- a/lib/gitlab/utils/override.rb
+++ b/lib/gitlab/utils/override.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_dependency 'gitlab/utils'
+
 module Gitlab
   module Utils
     module Override
diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb
index aa1f8e2fdda..3021a91dd83 100644
--- a/lib/gitlab/utils/strong_memoize.rb
+++ b/lib/gitlab/utils/strong_memoize.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_dependency 'gitlab/utils'
+
 module Gitlab
   module Utils
     module StrongMemoize
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index a42f02a84fd..7a42e4e92a0 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -1,13 +1,17 @@
 namespace :gitlab do
   namespace :assets do
     desc 'GitLab | Assets | Compile all frontend assets'
-    task compile: [
-      'yarn:check',
-      'gettext:po_to_json',
-      'rake:assets:precompile',
-      'webpack:compile',
-      'fix_urls'
-    ]
+    task :compile do
+      require_dependency 'gitlab/task_helpers'
+
+      %w[
+        yarn:check
+        gettext:po_to_json
+        rake:assets:precompile
+        webpack:compile
+        gitlab:assets:fix_urls
+      ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
+    end
 
     desc 'GitLab | Assets | Clean up old compiled frontend assets'
     task clean: ['rake:assets:clean']
diff --git a/scripts/clean-old-cached-assets b/scripts/clean-old-cached-assets
new file mode 100755
index 00000000000..7a3a62a477a
--- /dev/null
+++ b/scripts/clean-old-cached-assets
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Clean up cached files that are older than 1 week
+find tmp/cache/assets/sprockets/ -type f -mtime +7 -execdir rm -- "{}" \;
+
+du -d 0 -h tmp/cache/assets/sprockets | cut -f1 | xargs -I % echo "tmp/cache/assets/sprockets/ is currently %"