diff options
-rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 12 | ||||
-rwxr-xr-x | scripts/generate-gems-memory-metrics-static | 18 | ||||
-rwxr-xr-x | scripts/generate-gems-size-metrics-static | 30 | ||||
-rwxr-xr-x | scripts/memory-static | 20 | ||||
-rwxr-xr-x | scripts/memory-static-objects | 27 |
5 files changed, 54 insertions, 53 deletions
diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 50b843df585..2f3907a331a 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -5,15 +5,15 @@ memory-static: # Loads each of gems in the Gemfile and checks how much memory they consume when they are required. # 'derailed_benchmarks' internally uses 'get_process_mem' - - scripts/memory-static 'tmp/memory_static_full_report.txt' 'tmp/memory_static_metrics.txt' + - bundle exec derailed bundle:mem > tmp/memory_bundle_mem.txt + - scripts/generate-gems-size-metrics-static tmp/memory_bundle_mem.txt >> 'tmp/memory_metrics.txt' # Outputs detailed information about objects created while gems are loaded. # 'derailed_benchmarks' internally uses 'memory_profiler' - - scripts/memory-static-objects 'tmp/memory_static_objects_full_report.txt' 'tmp/memory_static_metrics.txt' + - bundle exec derailed bundle:objects > tmp/memory_bundle_objects.txt + - scripts/generate-gems-memory-metrics-static tmp/memory_bundle_objects.txt >> 'tmp/memory_metrics.txt' artifacts: paths: - - tmp/memory_static_full_report.txt - - tmp/memory_static_objects_full_report.txt - - tmp/memory_static_metrics.txt + - tmp/memory_*.txt reports: - metrics: tmp/memory_static_metrics.txt + metrics: tmp/memory_metrics.txt diff --git a/scripts/generate-gems-memory-metrics-static b/scripts/generate-gems-memory-metrics-static new file mode 100755 index 00000000000..aa7ce3615bf --- /dev/null +++ b/scripts/generate-gems-memory-metrics-static @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby + +abort "usage: #{__FILE__} <memory_bundle_objects_file_name>" unless ARGV.length == 1 +memory_bundle_objects_file_name = ARGV.first + +full_report = File.readlines(memory_bundle_objects_file_name) + +allocated_str = full_report[1] +retained_str = full_report[2] +allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str) +retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str) + +abort 'failed to process the benchmark output' unless allocated_stats && retained_stats + +puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}" +puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (1024 * 1024)).round(1)}" +puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}" +puts "memory_static_objects_retained_items #{retained_stats[:objects]}" diff --git a/scripts/generate-gems-size-metrics-static b/scripts/generate-gems-size-metrics-static new file mode 100755 index 00000000000..ceec8aaccf1 --- /dev/null +++ b/scripts/generate-gems-size-metrics-static @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby + +abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1 +memory_bundle_mem_file_name = ARGV.first + +full_report = File.readlines(memory_bundle_mem_file_name) + +def total_size(memory_bundle_mem_report) + stats = /TOP: (?<total_mibs_str>.*) MiB/.match(memory_bundle_mem_report.first) + abort 'failed to process the benchmark output' unless stats + "gem_total_size_mb #{stats[:total_mibs_str].to_f.round(1)}" +end + +TOP_LEVEL_GEM_LOG_FORMAT = /^ (?<gem_name>\S.*):\s*(?<gem_size>\d[.\d]*)\s*MiB/.freeze +def all_gems(memory_bundle_mem_report) + memory_bundle_mem_report.map do |line| + TOP_LEVEL_GEM_LOG_FORMAT.match(line) + end.compact +end + +def gems_as_metrics(gems_match_data) + gems_match_data.map do |gem| + gem_name = gem[:gem_name] + gem_size_mb = gem[:gem_size].to_f.round(1) + "gem_size_mb{name=\"#{gem_name}\"} #{gem_size_mb}" + end +end + +puts total_size(full_report) +puts gems_as_metrics(all_gems(full_report)).sort(&:casecmp) diff --git a/scripts/memory-static b/scripts/memory-static deleted file mode 100755 index 54f147a7a91..00000000000 --- a/scripts/memory-static +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env ruby - -require_relative '../lib/gitlab/popen' - -full_report_filename, metrics_filename = ARGV -abort 'usage: memory-static <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename - -full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:mem)) -abort 'failed to execute the benchmark' unless status.zero? - -File.open(full_report_filename, 'w') do |f| - f.write(full_report) -end - -stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report.lines.first) -abort 'failed to process the benchmark output' unless stats - -File.open(metrics_filename, 'a') do |f| - f.puts "memory_static_total_mb #{stats[:total_mibs_str].to_f.round(1)}" -end diff --git a/scripts/memory-static-objects b/scripts/memory-static-objects deleted file mode 100755 index 2ad38d9717c..00000000000 --- a/scripts/memory-static-objects +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env ruby - -require_relative '../lib/gitlab/popen' - -full_report_filename, metrics_filename = ARGV -abort 'usage: memory-static-objects <full_report_filename> <metrics_filename>' unless full_report_filename && metrics_filename - -full_report, status = Gitlab::Popen.popen(%w(bundle exec derailed bundle:objects)) -abort 'failed to execute the benchmark' unless status.zero? - -File.open(full_report_filename, 'w') do |f| - f.write(full_report) -end - -allocated_str = full_report.lines[1] -retained_str = full_report.lines[2] -allocated_stats = /Total allocated: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(allocated_str) -retained_stats = /Total retained: (?<bytes>.*) bytes \((?<objects>.*) objects\)/.match(retained_str) - -abort 'failed to process the benchmark output' unless allocated_stats && retained_stats - -File.open(metrics_filename, 'a') do |f| - f.puts "memory_static_objects_allocated_mb #{(allocated_stats[:bytes].to_f / (1024 * 1024)).round(1)}" - f.puts "memory_static_objects_retained_mb #{(retained_stats[:bytes].to_f / (104 * 1024)).round(1)}" - f.puts "memory_static_objects_allocated_items #{allocated_stats[:objects]}" - f.puts "memory_static_objects_retained_items #{retained_stats[:objects]}" -end |