summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKamil Trzciński <ayufan@ayufan.eu>2019-06-28 12:52:48 +0000
committerKamil Trzciński <ayufan@ayufan.eu>2019-06-28 12:52:48 +0000
commit423c4c43cb73c72e8b27d7c091f392e3a09be354 (patch)
tree4af4031686f4528d93cd73632df96a231de210b8
parent1dbdd3d4330c02de6546e57545d83fa3ce9d6ef1 (diff)
parentfb1d433703f902987a358af0d9cac12b64ec440e (diff)
downloadgitlab-ce-423c4c43cb73c72e8b27d7c091f392e3a09be354.tar.gz
Merge branch '62722-boot-app-in-ci-and-check-memory' into 'master'
Get memory data after booting app in production Closes #63585 See merge request gitlab-org/gitlab-ce!30031
-rw-r--r--.gitlab/ci/memory.gitlab-ci.yml23
-rw-r--r--config/boot.rb2
-rwxr-xr-xscripts/generate-memory-metrics-on-boot11
-rw-r--r--scripts/prepare_build.sh9
4 files changed, 40 insertions, 5 deletions
diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml
index 2f3907a331a..ffe5dbdc31b 100644
--- a/.gitlab/ci/memory.gitlab-ci.yml
+++ b/.gitlab/ci/memory.gitlab-ci.yml
@@ -17,3 +17,26 @@ memory-static:
- tmp/memory_*.txt
reports:
metrics: tmp/memory_metrics.txt
+
+# Show memory usage caused by invoking require per gem.
+# Unlike `memory-static`, it hits the app with one request to ensure that any last minute require-s have been called.
+# The application is booted in `production` environment.
+# All tests are run without a webserver (directly using Rack::Mock by default).
+memory-on-boot:
+ extends: .rspec-metadata-pg-10
+ variables:
+ NODE_ENV: "production"
+ RAILS_ENV: "production"
+ SETUP_DB: "true"
+ SKIP_STORAGE_VALIDATION: "true"
+ # we override the max_old_space_size to prevent OOM errors
+ NODE_OPTIONS: --max_old_space_size=3584
+ script:
+ # Both bootsnap and derailed monkey-patch Kernel#require, which leads to circular dependency
+ - DISABLE_BOOTSNAP=true PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> 'tmp/memory_on_boot.txt'
+ - scripts/generate-memory-metrics-on-boot tmp/memory_on_boot.txt >> 'tmp/memory_on_boot_metrics.txt'
+ artifacts:
+ paths:
+ - tmp/memory_*.txt
+ reports:
+ metrics: tmp/memory_on_boot_metrics.txt
diff --git a/config/boot.rb b/config/boot.rb
index 2811f0e6188..b76b26a5e75 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -3,7 +3,7 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
# Set up gems listed in the Gemfile.
require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
begin
- require 'bootsnap/setup'
+ require 'bootsnap/setup' unless ENV['DISABLE_BOOTSNAP']
rescue LoadError
# bootsnap is an optional dependency, so if we don't have it, it's fine
end
diff --git a/scripts/generate-memory-metrics-on-boot b/scripts/generate-memory-metrics-on-boot
new file mode 100755
index 00000000000..5197a8fcdcd
--- /dev/null
+++ b/scripts/generate-memory-metrics-on-boot
@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
+memory_bundle_mem_file_name = ARGV.first
+
+full_report = File.open(memory_bundle_mem_file_name).read
+
+stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report)
+abort 'failed to process the benchmark output' unless stats
+
+puts "total_memory_used_by_dependencies_on_boot_prod_env_mb #{stats[:total_mibs_str].to_f.round(1)}"
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 9b0d5d4f719..c1cade0a820 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -35,6 +35,7 @@ sed -i 's/username: root/username: gitlab/g' config/database.yml
if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
sed -i 's/localhost/postgres/g' config/database.yml
+ sed -i 's/username: git/username: postgres/g' config/database.yml
if [ -f config/database_geo.yml ]; then
sed -i 's/localhost/postgres/g' config/database_geo.yml
@@ -48,16 +49,16 @@ else # Assume it's mysql
fi
cp config/resque.yml.example config/resque.yml
-sed -i 's/localhost/redis/g' config/resque.yml
+sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
cp config/redis.cache.yml.example config/redis.cache.yml
-sed -i 's/localhost/redis/g' config/redis.cache.yml
+sed -i 's|url:.*$|url: redis://redis:6379/10|g' config/redis.cache.yml
cp config/redis.queues.yml.example config/redis.queues.yml
-sed -i 's/localhost/redis/g' config/redis.queues.yml
+sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
cp config/redis.shared_state.yml.example config/redis.shared_state.yml
-sed -i 's/localhost/redis/g' config/redis.shared_state.yml
+sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
if [ "$SETUP_DB" != "false" ]; then
setup_db