Gather memory usage data in tests

Log memory stats after running each spec file and compile the report.

5 files changed, 86 insertions, 0 deletions

diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 2d06a8acc58..c7a51beeee5 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -66,6 +66,8 @@
     - scripts/gitaly-test-spawn
     - date
     - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")'
+    - mkdir -p tmp/memory_test
+    - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
     - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
     - date
   artifacts:
@@ -77,6 +79,7 @@
       - rspec_flaky/
       - rspec_profiling/
       - tmp/capybara/
+      - tmp/memory_test/
 #    reports:
 #      junit: junit_rspec.xml
 
@@ -273,6 +276,7 @@ coverage:
   stage: post-test
   script:
     - bundle exec scripts/merge-simplecov
+    - bundle exec scripts/gather-test-memory-data
   coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
   artifacts:
     name: coverage
@@ -280,3 +284,4 @@ coverage:
     paths:
       - coverage/index.html
       - coverage/assets/
+      - tmp/memory_test/
diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data
new file mode 100755
index 00000000000..9992a83e6a6
--- /dev/null
+++ b/scripts/gather-test-memory-data
@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+
+def join_csv_files(output_path, input_paths)
+  return if input_paths.empty?
+
+  input_csvs = input_paths.map do |input_path|
+    CSV.read(input_path, headers: true)
+  end
+
+  CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv|
+    input_csvs.each do |input_csv|
+      input_csv.each do |line|
+        output_csv << line
+      end
+    end
+  end
+end
+
+join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort)
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 62fdc039b5e..21c36001a1b 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -103,6 +103,7 @@ RSpec.configure do |config|
   config.include RedisHelpers
   config.include Rails.application.routes.url_helpers, type: :routing
   config.include PolicyHelpers, type: :policy
+  config.include MemoryUsageHelper
 
   if ENV['CI']
     # This includes the first try, i.e. tests will be run 4 times before failing.
diff --git a/spec/support/helpers/memory_usage_helper.rb b/spec/support/helpers/memory_usage_helper.rb
new file mode 100644
index 00000000000..984ea8cc571
--- /dev/null
+++ b/spec/support/helpers/memory_usage_helper.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module MemoryUsageHelper
+  extend ActiveSupport::Concern
+
+  def gather_memory_data(csv_path)
+    write_csv_entry(csv_path,
+      {
+        example_group_path: TestEnv.topmost_example_group[:location],
+        example_group_description: TestEnv.topmost_example_group[:description],
+        time: Time.current,
+        job_name: ENV['CI_JOB_NAME']
+      }.merge(get_memory_usage))
+  end
+
+  def write_csv_entry(path, entry)
+    CSV.open(path, "a", headers: entry.keys, write_headers: !File.exist?(path)) do |file|
+      file << entry.values
+    end
+  end
+
+  def get_memory_usage
+    output, status = Gitlab::Popen.popen(%w(free -m))
+    abort "`free -m` return code is #{status}: #{output}" unless status.zero?
+
+    result = output.split("\n")[1].split(" ")[1..-1]
+    attrs = %i(m_total m_used m_free m_shared m_buffers_cache m_available).freeze
+
+    attrs.zip(result).to_h
+  end
+
+  included do |config|
+    config.after(:all) do
+      gather_memory_data(ENV['MEMORY_TEST_PATH']) if ENV['MEMORY_TEST_PATH']
+    end
+  end
+end
diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb
index e63099d89b7..893b10ea752 100644
--- a/spec/support/helpers/test_env.rb
+++ b/spec/support/helpers/test_env.rb
@@ -2,6 +2,7 @@ require 'rspec/mocks'
 require 'toml-rb'
 
 module TestEnv
+  extend ActiveSupport::Concern
   extend self
 
   ComponentFailedToInstallError = Class.new(StandardError)
@@ -108,6 +109,12 @@ module TestEnv
     setup_forked_repo
   end
 
+  included do |config|
+    config.append_before do
+      set_current_example_group
+    end
+  end
+
   def disable_mailer
     allow_any_instance_of(NotificationService).to receive(:mailer)
       .and_return(double.as_null_object)
@@ -297,8 +304,23 @@ module TestEnv
     FileUtils.rm_rf(path)
   end
 
+  def current_example_group
+    Thread.current[:current_example_group]
+  end
+
+  # looking for a top-level `describe`
+  def topmost_example_group
+    example_group = current_example_group
+    example_group = example_group[:parent_example_group] until example_group[:parent_example_group].nil?
+    example_group
+  end
+
   private
 
+  def set_current_example_group
+    Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
+  end
+
   # These are directories that should be preserved at cleanup time
   def test_dirs
     @test_dirs ||= %w[