summaryrefslogtreecommitdiff
path: root/lib/rspec_flaky
diff options
context:
space:
mode:
authorRémy Coutable <remy@rymai.me>2017-07-19 19:51:59 +0200
committerRémy Coutable <remy@rymai.me>2017-08-11 14:55:10 +0200
commit8f8fd342313b0cd459d2fedb5b461b0cc063f248 (patch)
tree87ea1fc1a405d1a0f50165e69e4f12508257d75a /lib/rspec_flaky
parentc62ae6cfd70d518386fdbcb9714d18ac4f5e8c31 (diff)
downloadgitlab-ce-8f8fd342313b0cd459d2fedb5b461b0cc063f248.tar.gz
Use a new RspecFlakyListener to detect flaky specs
Signed-off-by: Rémy Coutable <remy@rymai.me>
Diffstat (limited to 'lib/rspec_flaky')
-rw-r--r--lib/rspec_flaky/example.rb46
-rw-r--r--lib/rspec_flaky/flaky_example.rb39
-rw-r--r--lib/rspec_flaky/listener.rb75
3 files changed, 160 insertions, 0 deletions
diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb
new file mode 100644
index 00000000000..b6e790cbbab
--- /dev/null
+++ b/lib/rspec_flaky/example.rb
@@ -0,0 +1,46 @@
+module RspecFlaky
+ # This is a wrapper class for RSpec::Core::Example
+ class Example
+ delegate :status, :exception, to: :execution_result
+
+ def initialize(rspec_example)
+ @rspec_example = rspec_example.try(:example) || rspec_example
+ end
+
+ def uid
+ @uid ||= Digest::MD5.hexdigest("#{description}-#{file}")
+ end
+
+ def example_id
+ rspec_example.id
+ end
+
+ def file
+ metadata[:file_path]
+ end
+
+ def line
+ metadata[:line_number]
+ end
+
+ def description
+ metadata[:full_description]
+ end
+
+ def attempts
+ rspec_example.try(:attempts) || 1
+ end
+
+ private
+
+ attr_reader :rspec_example
+
+ def metadata
+ rspec_example.metadata
+ end
+
+ def execution_result
+ rspec_example.execution_result
+ end
+ end
+end
diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
new file mode 100644
index 00000000000..f81fb90e870
--- /dev/null
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -0,0 +1,39 @@
+module RspecFlaky
+ # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
+ class FlakyExample < OpenStruct
+ def initialize(example)
+ if example.respond_to?(:example_id)
+ super(
+ example_id: example.example_id,
+ file: example.file,
+ line: example.line,
+ description: example.description,
+ last_attempts_count: example.attempts,
+ flaky_reports: 1)
+ else
+ super
+ end
+ end
+
+ def first_flaky_at
+ self[:first_flaky_at] || Time.now
+ end
+
+ def last_flaky_at
+ Time.now
+ end
+
+ def last_flaky_job
+ return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
+
+ "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+ end
+
+ def to_h
+ super.merge(
+ first_flaky_at: first_flaky_at,
+ last_flaky_at: last_flaky_at,
+ last_flaky_job: last_flaky_job)
+ end
+ end
+end
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
new file mode 100644
index 00000000000..ec2fbd9e36c
--- /dev/null
+++ b/lib/rspec_flaky/listener.rb
@@ -0,0 +1,75 @@
+require 'json'
+
+module RspecFlaky
+ class Listener
+ attr_reader :all_flaky_examples, :new_flaky_examples
+
+ def initialize
+ @new_flaky_examples = {}
+ @all_flaky_examples = init_all_flaky_examples
+ end
+
+ def example_passed(notification)
+ current_example = RspecFlaky::Example.new(notification.example)
+
+ return unless current_example.attempts > 1
+
+ flaky_example_hash = all_flaky_examples[current_example.uid]
+
+ all_flaky_examples[current_example.uid] =
+ if flaky_example_hash
+ FlakyExample.new(flaky_example_hash).tap do |ex|
+ ex.last_attempts_count = current_example.attempts
+ ex.flaky_reports += 1
+ end
+ else
+ FlakyExample.new(current_example).tap do |ex|
+ new_flaky_examples[current_example.uid] = ex
+ end
+ end
+ end
+
+ def dump_summary(_)
+ write_report_file(all_flaky_examples, all_flaky_examples_report_path)
+
+ if new_flaky_examples.any?
+ Rails.logger.warn "\nNew flaky examples detected:\n"
+ Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
+
+ write_report_file(new_flaky_examples, new_flaky_examples_report_path)
+ end
+ end
+
+ def to_report(examples)
+ Hash[examples.map { |k, ex| [k, ex.to_h] }]
+ end
+
+ private
+
+ def init_all_flaky_examples
+ return {} unless File.exist?(all_flaky_examples_report_path)
+
+ all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path))
+
+ Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }]
+ end
+
+ def write_report_file(examples, file_path)
+ return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true'
+
+ report_path_dir = File.dirname(file_path)
+ FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir)
+ File.write(file_path, JSON.pretty_generate(to_report(examples)))
+ end
+
+ def all_flaky_examples_report_path
+ @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] ||
+ Rails.root.join("rspec_flaky/all-report.json")
+ end
+
+ def new_flaky_examples_report_path
+ @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] ||
+ Rails.root.join("rspec_flaky/new-report.json")
+ end
+ end
+end