summaryrefslogtreecommitdiff
path: root/scripts/static-analysis
blob: c6cf09e056be8e2c0d85dabc08ab0f99f5bfe6b1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
#!/usr/bin/env ruby
# frozen_string_literal: true

# We don't have auto-loading here
require_relative '../lib/gitlab'
require_relative '../lib/gitlab/popen'
require_relative '../lib/gitlab/popen/runner'

class StaticAnalysis
  ALLOWED_WARNINGS = [
    # https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329
    # warns if caniuse-lite package is older than 6 months. Ignore this
    # warning message so that GitLab backports don't fail.
    "Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`",
    # https://github.com/mime-types/mime-types-data/pull/50#issuecomment-1060908930
    "Type application/netcdf is already registered as a variant of application/netcdf."
  ].freeze

  Task = Struct.new(:command, :duration) do
    def cmd
      command.join(' ')
    end
  end
  NodeAssignment = Struct.new(:index, :tasks) do
    def total_duration
      return 0 if tasks.empty?

      tasks.sum(&:duration)
    end
  end

  def self.project_path
    project_root = File.expand_path('..', __dir__)

    if Gitlab.jh?
      "#{project_root}/jh"
    else
      project_root
    end
  end

  # `gettext:updated_check` and `gitlab:sidekiq:sidekiq_queues_yml:check` will fail on FOSS installations
  # (e.g. gitlab-org/gitlab-foss) since they test against a single
  # file that is generated by an EE installation, which can
  # contain values that a FOSS installation won't find. To work
  # around this we will only enable this task on EE installations.
  TASKS_WITH_DURATIONS_SECONDS = [
    (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
    Task.new(%w[yarn run lint:prettier], 200),
    Task.new(%w[bin/rake gettext:lint], 105),
    Task.new(%W[scripts/license-check.sh #{project_path}], 200),
    Task.new(%w[bin/rake lint:static_verification], 40),
    Task.new(%w[scripts/rubocop-max-files-in-cache-check], 25),
    Task.new(%w[bin/rake config_lint], 10),
    Task.new(%w[bin/rake gitlab:sidekiq:all_queues_yml:check], 15),
    (Gitlab.ee? ? Task.new(%w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check], 11) : nil),
    Task.new(%w[yarn run internal:stylelint], 8),
    Task.new(%w[scripts/lint-conflicts.sh], 1),
    Task.new(%w[yarn run block-dependencies], 1),
    Task.new(%w[yarn run check-dependencies], 1),
    Task.new(%w[scripts/lint-rugged], 1),
    Task.new(%w[scripts/gemfile_lock_changed.sh], 1),
    Task.new(%w[scripts/lint-vendored-gems.sh], 1)
  ].compact.freeze

  def run_tasks!(options = {})
    total_nodes = (ENV['CI_NODE_TOTAL'] || 1).to_i
    current_node_number = (ENV['CI_NODE_INDEX'] || 1).to_i
    node_assignment = tasks_to_run(total_nodes)[current_node_number - 1]

    if options[:dry_run]
      puts "Dry-run mode!"
      return
    end

    static_analysis = Gitlab::Popen::Runner.new
    start_time = Time.now
    static_analysis.run(node_assignment.tasks.map(&:command)) do |command, &run|
      task = node_assignment.tasks.find { |task| task.command == command }
      puts
      puts "$ #{task.cmd}"

      result = run.call

      puts "==> Finished in #{result.duration} seconds (expected #{task.duration} seconds)"
      puts
    end

    puts
    puts '==================================================='
    puts "Node finished running all tasks in #{Time.now - start_time} seconds (expected #{node_assignment.total_duration})"
    puts
    puts

    if static_analysis.all_success_and_clean?
      puts 'All static analyses passed successfully.'
    elsif static_analysis.all_success?
      puts 'All static analyses passed successfully, but we have warnings:'
      puts

      emit_warnings(static_analysis)

      exit 2 if warning_count(static_analysis).nonzero?
    else
      puts 'Some static analyses failed:'

      emit_warnings(static_analysis)
      emit_errors(static_analysis)

      exit 1
    end
  end

  def emit_warnings(static_analysis)
    static_analysis.warned_results.each do |result|
      puts
      puts "**** #{result.cmd.join(' ')} had the following warning(s):"
      puts
      puts result.stderr
      puts
    end
  end

  def emit_errors(static_analysis)
    static_analysis.failed_results.each do |result|
      puts
      puts "**** #{result.cmd.join(' ')} failed with the following error(s):"
      puts
      puts result.stdout
      puts result.stderr
      puts
    end
  end

  def warning_count(static_analysis)
    static_analysis.warned_results
      .count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) }
  end

  def tasks_to_run(node_total)
    total_time = TASKS_WITH_DURATIONS_SECONDS.sum(&:duration).to_f
    ideal_time_per_node = total_time / node_total
    tasks_by_duration_desc = TASKS_WITH_DURATIONS_SECONDS.sort_by { |a| -a.duration }
    nodes = Array.new(node_total) { |i| NodeAssignment.new(i + 1, []) }

    puts "Total expected time: #{total_time}; ideal time per job: #{ideal_time_per_node}.\n\n"
    puts "Tasks to distribute:"
    tasks_by_duration_desc.each { |task| puts "* #{task.cmd} (#{task.duration}s)" }

    # Distribute tasks optimally first
    puts "\nAssigning tasks optimally."
    distribute_tasks(tasks_by_duration_desc, nodes, ideal_time_per_node: ideal_time_per_node)

    # Distribute remaining tasks, ordered by ascending duration
    leftover_tasks = tasks_by_duration_desc - nodes.flat_map(&:tasks)

    if leftover_tasks.any?
      puts "\n\nAssigning remaining tasks: #{leftover_tasks.flat_map(&:cmd)}"
      distribute_tasks(leftover_tasks, nodes.sort_by { |node| node.total_duration })
    end

    nodes.each do |node|
      puts "\nExpected duration for node #{node.index}: #{node.total_duration} seconds"
      node.tasks.each { |task| puts "* #{task.cmd} (#{task.duration}s)" }
    end

    nodes
  end

  def distribute_tasks(tasks, nodes, ideal_time_per_node: nil)
    condition =
      if ideal_time_per_node
        ->(task, node, ideal_time_per_node) { (task.duration + node.total_duration) <= ideal_time_per_node }
      else
        ->(*) { true }
      end

    tasks.each do |task|
      nodes.each do |node|
        if condition.call(task, node, ideal_time_per_node)
          assign_task_to_node(tasks, node, task)
          break
        end
      end
    end
  end

  def assign_task_to_node(remaining_tasks, node, task)
    node.tasks << task
    puts "Assigning #{task.command} (#{task.duration}s) to node ##{node.index}. Node total duration: #{node.total_duration}s."
  end
end

if $PROGRAM_NAME == __FILE__
  options = {}

  if ARGV.include?('--dry-run')
    options[:dry_run] = true
  end

  StaticAnalysis.new.run_tasks!(options)
end