# frozen_string_literal: true require "influxdb-client" require "terminal-table" require "slack-notifier" require "colorize" module QA module Tools class ReliableReport include Support::InfluxdbTools include Support::API RELIABLE_REPORT_LABEL = "reliable test report" # Project for report creation: https://gitlab.com/gitlab-org/gitlab PROJECT_ID = 278964 def initialize(range) @range = range.to_i @slack_channel = "#quality-reports" end # Run reliable reporter # # @param [Integer] range amount of days for results range # @param [String] report_in_issue_and_slack # @return [void] def self.run(range: 14, report_in_issue_and_slack: "false") reporter = new(range) reporter.print_report if report_in_issue_and_slack == "true" reporter.report_in_issue_and_slack reporter.close_previous_reports end rescue StandardError => e reporter&.notify_failure(e) raise(e) end # Print top stable specs # # @return [void] def print_report puts "#{stable_summary_table}\n\n" stable_results_tables.each { |stage, table| puts "#{table}\n\n" } return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty? puts "#{unstable_summary_table}\n\n" unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" } end # Create report issue # # @return [void] def report_in_issue_and_slack puts "Creating report".colorize(:green) issue = api_update( :post, "projects/#{PROJECT_ID}/issues", title: "Reliable e2e test report", description: report_issue_body, labels: "#{RELIABLE_REPORT_LABEL},Quality,test,type::maintenance,automation:ml" ) @report_iid = issue[:iid] web_url = issue[:web_url] puts "Created report issue: #{web_url}" puts "Sending slack notification".colorize(:green) notifier.post( icon_emoji: ":tanuki-protect:", text: <<~TEXT ```#{stable_summary_table}``` ```#{unstable_summary_table}``` #{web_url} TEXT ) puts "Done!" end # Close previous reliable test reports # # @return [void] def close_previous_reports puts "Closing previous reports".colorize(:green) issues = api_get("projects/#{PROJECT_ID}/issues?labels=#{RELIABLE_REPORT_LABEL}&state=opened") issues .reject { |issue| issue[:iid] == report_iid } .each do |issue| issue_iid = issue[:iid] issue_endpoint = "projects/#{PROJECT_ID}/issues/#{issue_iid}" puts "Closing previous report '#{issue[:web_url]}'" api_update(:put, issue_endpoint, state_event: "close") api_update(:post, "#{issue_endpoint}/notes", body: "Closed issue in favor of ##{report_iid}") end end # Notify failure # # @param [StandardError] error # @return [void] def notify_failure(error) notifier.post( text: "Reliable reporter failed to create report. Error: ```#{error}```", icon_emoji: ":sadpanda:" ) end private attr_reader :range, :slack_channel, :report_iid # Slack notifier # # @return [Slack::Notifier] def notifier @notifier ||= Slack::Notifier.new( slack_webhook_url, channel: slack_channel, username: "Reliable Spec Report" ) end # Gitlab access token # # @return [String] def gitlab_access_token @gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable") end # Gitlab api url # # @return [String] def gitlab_api_url @gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable") end # Slack webhook url # # @return [String] def slack_webhook_url @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") end # Markdown formatted report issue body # # @return [String] def report_issue_body execution_interval = "(#{Date.today - range} - #{Date.today})" issue = [] issue << "[[_TOC_]]" issue << "# Candidates for promotion to reliable #{execution_interval}" issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**" issue << stable_summary_table(markdown: true).to_s issue << results_markdown(:stable) return issue.join("\n\n") if unstable_reliable_test_runs.empty? issue << "# Reliable specs with failures #{execution_interval}" issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**" issue << unstable_summary_table(markdown: true).to_s issue << results_markdown(:unstable) issue.join("\n\n") end # Stable spec summary table # # @param [Boolean] markdown # @return [Terminal::Table] def stable_summary_table(markdown: false) terminal_table( rows: stable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Stable spec summary for past #{range} days".ljust(50), headings: %w[STAGE COUNT], markdown: markdown ) end # Unstable reliable summary table # # @param [Boolean] markdown # @return [Terminal::Table] def unstable_summary_table(markdown: false) terminal_table( rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] }, title: "Unstable spec summary for past #{range} days".ljust(50), headings: %w[STAGE COUNT], markdown: markdown ) end # Result tables for stable specs # # @param [Boolean] markdown # @return [Hash] def stable_results_tables(markdown: false) results_tables(:stable, markdown: markdown) end # Result table for unstable specs # # @param [Boolean] markdown # @return [Hash] def unstable_reliable_results_tables(markdown: false) results_tables(:unstable, markdown: markdown) end # Markdown formatted tables # # @param [Symbol] type result type - :stable, :unstable # @return [String] def results_markdown(type) runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs results_tables(type, markdown: true).map do |stage, table| <<~STAGE.strip ## #{stage} (#{runs[stage].count})
Executions table #{table}
STAGE end.join("\n\n") end # Results table # # @param [Symbol] type result type - :stable, :unstable # @param [Boolean] markdown # @return [Hash] def results_tables(type, markdown: false) (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs| headings = ["name", "runs", "failures", "failure rate"] [stage, terminal_table( title: "Top #{type} specs in '#{stage}' stage for past #{range} days", headings: headings.map(&:upcase), markdown: markdown, rows: specs.map do |k, v| [name_column(name: k, file: v[:file], markdown: markdown), *table_params(v.values)] end )] end end # Stable specs # # @return [Hash] def stable_test_runs @top_stable ||= begin stable_specs = test_runs(reliable: false).transform_values do |specs| specs .reject { |k, v| v[:failure_rate] != 0 } .sort_by { |k, v| -v[:runs] } .to_h end stable_specs.reject { |k, v| v.empty? } end end # Unstable reliable specs # # @return [Hash] def unstable_reliable_test_runs @top_unstable_reliable ||= begin unstable = test_runs(reliable: true).transform_values do |specs| specs .reject { |k, v| v[:failure_rate] == 0 } .sort_by { |k, v| -v[:failure_rate] } .to_h end unstable.reject { |k, v| v.empty? } end end # Terminal table for result formatting # # @param [Array] rows # @param [Array] headings # @param [String] title # @param [Boolean] markdown # @return [Terminal::Table] def terminal_table(rows:, headings:, title:, markdown:) Terminal::Table.new( headings: headings, title: markdown ? nil : title, rows: rows, style: markdown ? { border: :markdown } : { all_separators: true } ) end # Spec parameters for table row # # @param [Array] parameters # @return [Array] def table_params(parameters) [*parameters[1..2], "#{parameters.last}%"] end # Name column content # # @param [String] name # @param [String] file # @param [Boolean] markdown # @return [String] def name_column(name:, file:, markdown: false) return "**name**: #{name}
**file**: #{file}" if markdown wrapped_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name "name: '#{wrapped_name}'\nfile: #{file.ljust(160)}" end # Test executions grouped by name # # @param [Boolean] reliable # @return [Hash] def test_runs(reliable:) puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green)) all_runs = query_api.query(query: query(reliable)).values all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result| records = table.records.sort_by { |record| record.values["_time"] } # skip specs that executed less time than defined by range or stopped executing before report date # offset 1 day due to how schedulers are configured and first run can be 1 day later next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1) next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1 last_record = records.last.values name = last_record["name"] file = last_record["file_path"].split("/").last stage = last_record["stage"] || "unknown" runs = records.count failed = records.count { |r| r.values["status"] == "failed" } failure_rate = (failed.to_f / runs.to_f) * 100 result[stage][name] = { file: file, runs: runs, failed: failed, failure_rate: failure_rate == 0 ? failure_rate.round(0) : failure_rate.round(2) } end end # Flux query # # @param [Boolean] reliable # @return [String] def query(reliable) <<~QUERY from(bucket: "#{Support::InfluxdbTools::INFLUX_TEST_METRICS_BUCKET}") |> range(start: -#{range}d) |> filter(fn: (r) => r._measurement == "test-stats") |> filter(fn: (r) => r.run_type == "staging-full" or r.run_type == "staging-sanity" or r.run_type == "staging-sanity-no-admin" or r.run_type == "production-full" or r.run_type == "production-sanity" or r.run_type == "package-and-qa" or r.run_type == "nightly" ) |> filter(fn: (r) => r.status != "pending" and r.merge_request == "false" and r.quarantined == "false" and r.smoke == "false" and r.reliable == "#{reliable}" and r._field == "id" ) |> group(columns: ["name"]) QUERY end # Api get request # # @param [String] path # @param [Hash] payload # @return [Hash, Array] def api_get(path) response = get("#{gitlab_api_url}/#{path}", { headers: { "PRIVATE-TOKEN" => gitlab_access_token } }) parse_body(response) end # Api update request # # @param [Symbol] verb :post or :put # @param [String] path # @param [Hash] payload # @return [Hash, Array] def api_update(verb, path, **payload) response = send( verb, "#{gitlab_api_url}/#{path}", payload, { headers: { "PRIVATE-TOKEN" => gitlab_access_token } } ) parse_body(response) end end end end