From 31fc5965baf9ca23a1da4a0ffc350102fe4c89c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 30 Jan 2019 12:28:39 +0100 Subject: Monkey-patch Sprockets to prevent cache ballooning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .../initializers/sprockets_base_file_digest_key.rb | 3 +++ lib/gitlab/patch/sprockets_base_file_digest_key.rb | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 config/initializers/sprockets_base_file_digest_key.rb create mode 100644 lib/gitlab/patch/sprockets_base_file_digest_key.rb diff --git a/config/initializers/sprockets_base_file_digest_key.rb b/config/initializers/sprockets_base_file_digest_key.rb new file mode 100644 index 00000000000..81ff3812091 --- /dev/null +++ b/config/initializers/sprockets_base_file_digest_key.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +Sprockets::Base.prepend(Gitlab::Patch::SprocketsBaseFileDigestKey) diff --git a/lib/gitlab/patch/sprockets_base_file_digest_key.rb b/lib/gitlab/patch/sprockets_base_file_digest_key.rb new file mode 100644 index 00000000000..3925cdbbada --- /dev/null +++ b/lib/gitlab/patch/sprockets_base_file_digest_key.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# This monkey patch prevent cache ballooning when caching tmp/cache/assets/sprockets +# on the CI. See https://github.com/rails/sprockets/issues/563 and +# https://github.com/rails/sprockets/compare/3.x...jmreid:no-mtime-for-digest-key. +module Gitlab + module Patch + module SprocketsBaseFileDigestKey + def file_digest(path) + if stat = self.stat(path) + digest = self.stat_digest(path, stat) + integrity_uri = self.hexdigest_integrity_uri(digest) + + key = Sprockets::UnloadedAsset.new(path, self).file_digest_key(integrity_uri) + cache.fetch(key) do + digest + end + end + end + end + end +end -- cgit v1.2.1 From 6b39f55d3f0ba0d48e920693c4d5128e26769a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 21 Jan 2019 20:20:38 +0100 Subject: Add cache to the 'compile-assets' and 'gitlab:assets:compile' jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 25 +++++++++++++++++-------- lib/gitlab/task_helpers.rb | 8 +++++++- lib/gitlab/utils/merge_hash.rb | 2 ++ lib/gitlab/utils/override.rb | 2 ++ lib/gitlab/utils/strong_memoize.rb | 2 ++ lib/tasks/gitlab/assets.rake | 18 +++++++++++------- scripts/clean-old-cached-assets | 6 ++++++ 7 files changed, 47 insertions(+), 16 deletions(-) create mode 100755 scripts/clean-old-cached-assets diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee9eaeae723..b93a79de994 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -386,20 +386,25 @@ flaky-examples-check: - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT +.assets-compile-cache: &assets-compile-cache + cache: + key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v3" + paths: + - vendor/ruby/ + - .yarn-cache/ + - tmp/cache/assets/sprockets + compile-assets: <<: *dedicated-runner <<: *except-docs <<: *use-pg stage: prepare - cache: - <<: *default-cache script: - node --version - - date - yarn install --frozen-lockfile --cache-folder .yarn-cache - - date - free -m - bundle exec rake gitlab:assets:compile + - scripts/clean-old-cached-assets variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -408,6 +413,7 @@ compile-assets: paths: - node_modules - public/assets + <<: *assets-compile-cache setup-test-env: <<: *dedicated-runner @@ -628,7 +634,9 @@ gitlab:setup-mysql: gitlab:assets:compile: <<: *dedicated-no-docs-pull-cache-job image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1 - dependencies: [] + dependencies: + - setup-test-env + - compile-assets services: - docker:stable-dind variables: @@ -642,18 +650,19 @@ gitlab:assets:compile: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 script: - - date + - node --version - yarn install --frozen-lockfile --production --cache-folder .yarn-cache - - date - free -m - bundle exec rake gitlab:assets:compile - - scripts/build_assets_image + - time scripts/build_assets_image + - scripts/clean-old-cached-assets artifacts: name: webpack-report expire_in: 31d paths: - webpack-report/ - public/assets/ + <<: *assets-compile-cache only: - //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ee diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 224bb648d8f..8532845f3cb 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require 'gitlab/utils/strong_memoize' +require_dependency 'gitlab/utils/strong_memoize' # rubocop:disable Rails/Output module Gitlab @@ -13,6 +13,12 @@ module Gitlab extend self + def invoke_and_time_task(task) + start = Time.now + Rake::Task[task].invoke + puts "`#{task}` finished in #{Time.now - start} seconds" + end + # Ask if the user wants to continue # # Returns "yes" the user chose to continue diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb index fc237861e2f..48ba13b8561 100644 --- a/lib/gitlab/utils/merge_hash.rb +++ b/lib/gitlab/utils/merge_hash.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module MergeHash diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c87e97d0213..f5299439fce 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module Override diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index aa1f8e2fdda..3021a91dd83 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module StrongMemoize diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index a42f02a84fd..7a42e4e92a0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -1,13 +1,17 @@ namespace :gitlab do namespace :assets do desc 'GitLab | Assets | Compile all frontend assets' - task compile: [ - 'yarn:check', - 'gettext:po_to_json', - 'rake:assets:precompile', - 'webpack:compile', - 'fix_urls' - ] + task :compile do + require_dependency 'gitlab/task_helpers' + + %w[ + yarn:check + gettext:po_to_json + rake:assets:precompile + webpack:compile + gitlab:assets:fix_urls + ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) + end desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/scripts/clean-old-cached-assets b/scripts/clean-old-cached-assets new file mode 100755 index 00000000000..7a3a62a477a --- /dev/null +++ b/scripts/clean-old-cached-assets @@ -0,0 +1,6 @@ +#!/bin/bash + +# Clean up cached files that are older than 1 week +find tmp/cache/assets/sprockets/ -type f -mtime +7 -execdir rm -- "{}" \; + +du -d 0 -h tmp/cache/assets/sprockets | cut -f1 | xargs -I % echo "tmp/cache/assets/sprockets/ is currently %" -- cgit v1.2.1