diff options
Diffstat (limited to 'tooling/rspec_flaky')
-rw-r--r-- | tooling/rspec_flaky/config.rb | 27 | ||||
-rw-r--r-- | tooling/rspec_flaky/example.rb | 53 | ||||
-rw-r--r-- | tooling/rspec_flaky/flaky_example.rb | 40 | ||||
-rw-r--r-- | tooling/rspec_flaky/flaky_examples_collection.rb | 38 | ||||
-rw-r--r-- | tooling/rspec_flaky/listener.rb | 67 | ||||
-rw-r--r-- | tooling/rspec_flaky/report.rb | 56 |
6 files changed, 281 insertions, 0 deletions
diff --git a/tooling/rspec_flaky/config.rb b/tooling/rspec_flaky/config.rb new file mode 100644 index 00000000000..ea18a601c11 --- /dev/null +++ b/tooling/rspec_flaky/config.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module RspecFlaky + class Config + def self.generate_report? + !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) + end + + def self.suite_flaky_examples_report_path + ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/suite-report.json") + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/report.json") + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/new-report.json") + end + + def self.rails_path(path) + return path unless defined?(Rails) + + Rails.root.join(path) + end + end +end diff --git a/tooling/rspec_flaky/example.rb b/tooling/rspec_flaky/example.rb new file mode 100644 index 00000000000..18f8c5acc1c --- /dev/null +++ b/tooling/rspec_flaky/example.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'forwardable' +require 'digest' + +module RspecFlaky + # This is a wrapper class for RSpec::Core::Example + class Example + extend Forwardable + + def_delegators :execution_result, :status, :exception + + def initialize(rspec_example) + @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example + end + + def uid + @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") + end + + def example_id + rspec_example.id + end + + def file + metadata[:file_path] + end + + def line + metadata[:line_number] + end + + def description + metadata[:full_description] + end + + def attempts + rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1 + end + + private + + attr_reader :rspec_example + + def metadata + rspec_example.metadata + end + + def execution_result + rspec_example.execution_result + end + end +end diff --git a/tooling/rspec_flaky/flaky_example.rb b/tooling/rspec_flaky/flaky_example.rb new file mode 100644 index 00000000000..4f3688dbeed --- /dev/null +++ b/tooling/rspec_flaky/flaky_example.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'ostruct' + +module RspecFlaky + # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file + class FlakyExample < OpenStruct + def initialize(example) + if example.respond_to?(:example_id) + super( + example_id: example.example_id, + file: example.file, + line: example.line, + description: example.description, + last_attempts_count: example.attempts, + flaky_reports: 0) + else + super + end + end + + def update_flakiness!(last_attempts_count: nil) + self.first_flaky_at ||= Time.now + self.last_flaky_at = Time.now + self.flaky_reports += 1 + self.last_attempts_count = last_attempts_count if last_attempts_count + + if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end + end + + def to_h + super.merge( + first_flaky_at: first_flaky_at, + last_flaky_at: last_flaky_at, + last_flaky_job: last_flaky_job) + end + end +end diff --git a/tooling/rspec_flaky/flaky_examples_collection.rb b/tooling/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..acbfb411873 --- /dev/null +++ b/tooling/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'active_support/hash_with_indifferent_access' +require 'delegate' + +require_relative 'flaky_example' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def initialize(collection = {}) + unless collection.is_a?(Hash) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" + end + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_h + transform_values { |example| example.to_h }.deep_symbolize_keys + end + + def -(other) + unless other.respond_to?(:key) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" + end + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/tooling/rspec_flaky/listener.rb b/tooling/rspec_flaky/listener.rb new file mode 100644 index 00000000000..a5c68d830db --- /dev/null +++ b/tooling/rspec_flaky/listener.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'json' + +require_relative 'config' +require_relative 'example' +require_relative 'flaky_example' +require_relative 'flaky_examples_collection' +require_relative 'report' + +module RspecFlaky + class Listener + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = RspecFlaky::FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) + end + + def example_passed(notification) + current_example = RspecFlaky::Example.new(notification.example) + + return unless current_example.attempts > 1 + + flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example + end + + def dump_summary(_) + RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) + # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + + new_flaky_examples = flaky_examples - suite_flaky_examples + if new_flaky_examples.any? + rails_logger_warn("\nNew flaky examples detected:\n") + rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) # rubocop:disable Gitlab/Json + + RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path) + # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) + end + end + + private + + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + if suite_flaky_examples_json + RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples + else + return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) + + RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples + end + end + + def rails_logger_warn(text) + target = defined?(Rails) ? Rails.logger : Kernel + + target.warn(text) + end + end +end diff --git a/tooling/rspec_flaky/report.rb b/tooling/rspec_flaky/report.rb new file mode 100644 index 00000000000..3acfe7d2125 --- /dev/null +++ b/tooling/rspec_flaky/report.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'json' +require 'time' + +require_relative 'config' +require_relative 'flaky_examples_collection' + +module RspecFlaky + # This class is responsible for loading/saving JSON reports, and pruning + # outdated examples. + class Report < SimpleDelegator + OUTDATED_DAYS_THRESHOLD = 7 + + attr_reader :flaky_examples + + def self.load(file_path) + load_json(File.read(file_path)) + end + + def self.load_json(json) + new(RspecFlaky::FlakyExamplesCollection.new(JSON.parse(json))) + end + + def initialize(flaky_examples) + unless flaky_examples.is_a?(RspecFlaky::FlakyExamplesCollection) + raise ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" + end + + @flaky_examples = flaky_examples + super(flaky_examples) + end + + def write(file_path) + unless RspecFlaky::Config.generate_report? + Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" + return + end + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) + + File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) + end + + def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) + outdated_date_threshold = Time.now - (3600 * 24 * days) + updated_hash = flaky_examples.dup + .delete_if do |uid, hash| + hash[:last_flaky_at] && Time.parse(hash[:last_flaky_at]).to_i < outdated_date_threshold.to_i + end + + self.class.new(RspecFlaky::FlakyExamplesCollection.new(updated_hash)) + end + end +end |