Add latest changes from gitlab-org/gitlab@15-8-stable-ee

7 files changed, 284 insertions, 9 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 595a41fd8ca..5bb29a7aa3b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,6 +59,8 @@ workflow:
         CRYSTALBALL: "true"
         CREATE_INCIDENT_FOR_PIPELINE_FAILURE: "true"
         NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
+        BROKEN_BRANCH_INCIDENTS_PROJECT: "gitlab-org/quality/engineering-productivity/master-broken-incidents"
+        BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN: "${BROKEN_MASTER_INCIDENTS_PROJECT_TOKEN}"
         PIPELINE_NAME: 'Scheduled $CI_COMMIT_BRANCH pipeline'
     # Run pipelines for ruby2 branch
     - if: '$CI_COMMIT_BRANCH == "ruby2" && $CI_PIPELINE_SOURCE == "schedule"'
@@ -76,6 +78,8 @@ workflow:
         GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
         CREATE_INCIDENT_FOR_PIPELINE_FAILURE: "true"
         NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
+        BROKEN_BRANCH_INCIDENTS_PROJECT: "gitlab-org/quality/engineering-productivity/master-broken-incidents"
+        BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN: "${BROKEN_MASTER_INCIDENTS_PROJECT_TOKEN}"
     # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       variables:
@@ -84,6 +88,8 @@ workflow:
         OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY3"
         CREATE_INCIDENT_FOR_PIPELINE_FAILURE: "true"
         NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
+        BROKEN_BRANCH_INCIDENTS_PROJECT: "gitlab-org/quality/engineering-productivity/master-broken-incidents"
+        BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN: "${BROKEN_MASTER_INCIDENTS_PROJECT_TOKEN}"
     # For tags, create a pipeline.
     - if: '$CI_COMMIT_TAG'
       variables:
@@ -96,6 +102,9 @@ workflow:
       variables:
         RUBY_VERSION: "2.7"
         NOTIFY_PIPELINE_FAILURE_CHANNEL: "releases"
+        CREATE_INCIDENT_FOR_PIPELINE_FAILURE: "true"
+        BROKEN_BRANCH_INCIDENTS_PROJECT: "gitlab-org/release/tasks"
+        BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN: "${BROKEN_STABLE_INCIDENTS_PROJECT_TOKEN}"
     - if: '$CI_COMMIT_BRANCH =~ /^\d+-\d+-auto-deploy-\d+$/'
       variables:
         RUBY_VERSION: "2.7"
diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml
index 84fb5a55ed1..638e1cd8bd8 100644
--- a/.gitlab/ci/notify.gitlab-ci.yml
+++ b/.gitlab/ci/notify.gitlab-ci.yml
@@ -43,8 +43,9 @@ notify-pipeline-failure:
     - .notify:rules:notify-pipeline-failure
   image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
   variables:
-    BROKEN_MASTER_INCIDENTS_PROJECT: "gitlab-org/quality/engineering-productivity/master-broken-incidents"
-    BROKEN_MASTER_INCIDENT_JSON: "${CI_PROJECT_DIR}/incident.json"
+    INCIDENT_PROJECT: "${BROKEN_BRANCH_INCIDENTS_PROJECT}"
+    BROKEN_BRANCH_PROJECT_TOKEN: "${BROKEN_BRANCH_INCIDENTS_PROJECT_TOKEN}"
+    INCIDENT_JSON: "${CI_PROJECT_DIR}/incident.json"
     SLACK_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}"
     FAILED_PIPELINE_SLACK_MESSAGE_FILE: "${CI_PROJECT_DIR}/failed_pipeline_slack_message.json"
   before_script:
@@ -54,17 +55,17 @@ notify-pipeline-failure:
   script:
     - |
       if [[ "${CREATE_INCIDENT_FOR_PIPELINE_FAILURE}" == "true" ]]; then
-        scripts/create-pipeline-failure-incident.rb -p ${BROKEN_MASTER_INCIDENTS_PROJECT} -f ${BROKEN_MASTER_INCIDENT_JSON} -t ${BROKEN_MASTER_INCIDENTS_PROJECT_TOKEN};
-        echosuccess "Created incident $(jq '.web_url' ${BROKEN_MASTER_INCIDENT_JSON})";
+        scripts/create-pipeline-failure-incident.rb -p ${INCIDENT_PROJECT} -f ${INCIDENT_JSON} -t ${BROKEN_BRANCH_PROJECT_TOKEN};
+        echosuccess "Created incident $(jq '.web_url' ${INCIDENT_JSON})";
       fi
     - |
-      scripts/generate-failed-pipeline-slack-message.rb -i ${BROKEN_MASTER_INCIDENT_JSON} -f ${FAILED_PIPELINE_SLACK_MESSAGE_FILE};
+      scripts/generate-failed-pipeline-slack-message.rb -i ${INCIDENT_JSON} -f ${FAILED_PIPELINE_SLACK_MESSAGE_FILE};
       curl -X POST -H 'Content-Type: application/json' --data @${FAILED_PIPELINE_SLACK_MESSAGE_FILE} "$CI_SLACK_WEBHOOK_URL" ||
         scripts/slack ${SLACK_CHANNEL} "☠️  Broken pipeline notification failed! ☠️  See ${CI_JOB_URL}" ci_failing "Failed pipeline reporter"
 
   artifacts:
     paths:
-      - ${BROKEN_MASTER_INCIDENT_JSON}
+      - ${INCIDENT_JSON}
       - ${FAILED_PIPELINE_SLACK_MESSAGE_FILE}
     when: always
     expire_in: 2 days
diff --git a/scripts/api/commit_merge_requests.rb b/scripts/api/commit_merge_requests.rb
new file mode 100644
index 00000000000..3cf8dc87497
--- /dev/null
+++ b/scripts/api/commit_merge_requests.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative 'default_options'
+
+class CommitMergeRequests
+  def initialize(options)
+    @project = options.fetch(:project)
+    @sha = options.fetch(:sha)
+
+    # If api_token is nil, it's set to '' to allow unauthenticated requests (for forks).
+    api_token = options.fetch(:api_token, '')
+
+    warn "No API token given." if api_token.empty?
+
+    @client = Gitlab.client(
+      endpoint: options.fetch(:endpoint, API::DEFAULT_OPTIONS[:endpoint]),
+      private_token: api_token
+    )
+  end
+
+  def execute
+    client.commit_merge_requests(project, sha)
+  end
+
+  private
+
+  attr_reader :project, :sha, :client
+end
diff --git a/scripts/create-pipeline-failure-incident.rb b/scripts/create-pipeline-failure-incident.rb
index 1035a680291..2b86fac680d 100755
--- a/scripts/create-pipeline-failure-incident.rb
+++ b/scripts/create-pipeline-failure-incident.rb
@@ -5,9 +5,10 @@
 require 'optparse'
 require 'json'
 
-require_relative 'api/pipeline_failed_jobs'
+require_relative 'api/commit_merge_requests'
 require_relative 'api/create_issue'
 require_relative 'api/create_issue_discussion'
+require_relative 'api/pipeline_failed_jobs'
 
 class CreatePipelineFailureIncident
   DEFAULT_OPTIONS = {
@@ -29,6 +30,8 @@ class CreatePipelineFailureIncident
       labels: incident_labels
     }
 
+    payload[:assignee_ids] = assignee_ids if stable_branch_incident?
+
     CreateIssue.new(project: project, api_token: api_token).execute(payload).tap do |incident|
       CreateIssueDiscussion.new(project: project, api_token: api_token)
         .execute(issue_iid: incident.iid, body: "## Root Cause Analysis")
@@ -41,8 +44,18 @@ class CreatePipelineFailureIncident
 
   attr_reader :project, :api_token
 
+  def stable_branch_incident?
+    ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
+  end
+
   def failed_jobs
-    @failed_jobs ||= PipelineFailedJobs.new(API::DEFAULT_OPTIONS.dup.merge(exclude_allowed_to_fail_jobs: true)).execute
+    @failed_jobs ||= PipelineFailedJobs.new(API::DEFAULT_OPTIONS.merge(exclude_allowed_to_fail_jobs: true)).execute
+  end
+
+  def merge_request
+    @merge_request ||= CommitMergeRequests.new(
+      API::DEFAULT_OPTIONS.merge(sha: ENV['CI_COMMIT_SHA'])
+    ).execute.first
   end
 
   def now
@@ -63,6 +76,12 @@ class CreatePipelineFailureIncident
   end
 
   def description
+    return broken_stable_description_content if stable_branch_incident?
+
+    broken_master_description_content
+  end
+
+  def broken_master_description_content
     <<~MARKDOWN
     ## #{project_link} pipeline #{pipeline_link} failed
 
@@ -107,7 +126,62 @@ class CreatePipelineFailureIncident
     MARKDOWN
   end
 
+  def broken_stable_description_content
+    <<~MARKDOWN
+    ## #{project_link} pipeline #{pipeline_link} failed
+
+    **Branch: #{branch_link}**
+
+    **Commit: #{commit_link}**
+
+    **Merge Request: #{merge_request_link}**
+
+    **Triggered by** #{triggered_by_link} • **Source:** #{source} • **Duration:** #{pipeline_duration} minutes
+
+    **Failed jobs (#{failed_jobs.size}):**
+
+    #{failed_jobs_list}
+
+    ### General guidelines
+
+    A broken stable branch prevents patch releases from being built.
+    Fixing the pipeline is a priority to prevent any delays in releases.
+
+    The process in the [Broken `master` handbook guide](https://about.gitlab.com/handbook/engineering/workflow/#broken-master) can be referenced since much of that process also applies here.
+
+    ### Investigation
+
+    **Be sure to fill the `Timeline` for this incident.**
+
+    1. If the failure is new, and looks like a potential flaky failure, you can retry the failing job.
+      Make sure to mention the retry in the `Timeline` and leave a link to the retried job.
+    1. Search for similar master-broken issues in https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues
+      1. If one exists, ask the DRI of the master-broken issue to cherry-pick any resulting merge requests into the stable branch
+
+    @gitlab-org/release/managers if the merge request author or maintainer is not available, this can be escalated using the dev-on-call process in the [#dev-escalation slack channel](https://gitlab.slack.com/archives/CLKLMSUR4).
+
+    ### Pre-resolution
+
+    If you believe that there's an easy resolution by either:
+
+    - Reverting a particular merge request.
+    - Making a quick fix (for example, one line or a few similar simple changes in a few lines).
+      You can create a merge request, assign to any available maintainer, and ping people that were involved/related to the introduction of the failure.
+      Additionally, a message can be posted in `#backend_maintainers` or `#frontend_maintainers` to get a maintainer take a look at the fix ASAP.
+    - Cherry picking a change that was used to fix a similar master-broken issue.
+
+    In both cases, make sure to add the ~"pipeline:expedite" label to speed up the `stable`-fixing pipelines.
+
+    ### Resolution
+
+    Add a comment to this issue describing how this incident could have been prevented earlier in the Merge Request pipeline (rather than the merge commit pipeline).
+
+    MARKDOWN
+  end
+
   def incident_labels
+    return ['release-blocker'] if stable_branch_incident?
+
     master_broken_label =
       if ENV['CI_PROJECT_NAME'] == 'gitlab-foss'
         'master:foss-broken'
@@ -118,6 +192,12 @@ class CreatePipelineFailureIncident
     DEFAULT_LABELS.dup << master_broken_label
   end
 
+  def assignee_ids
+    ids = [ENV['GITLAB_USER_ID'].to_i]
+    ids << merge_request['author']['id'].to_i if merge_request
+    ids
+  end
+
   def pipeline_link
     "[##{ENV['CI_PIPELINE_ID']}](#{ENV['CI_PIPELINE_URL']})"
   end
@@ -134,6 +214,12 @@ class CreatePipelineFailureIncident
     "[#{ENV['CI_COMMIT_TITLE']}](#{ENV['CI_PROJECT_URL']}/-/commit/#{ENV['CI_COMMIT_SHA']})"
   end
 
+  def merge_request_link
+    return 'N/A' unless merge_request
+
+    "[#{merge_request['title']}](#{merge_request['web_url']})"
+  end
+
   def source
     "`#{ENV['CI_PIPELINE_SOURCE']}`"
   end
diff --git a/scripts/generate-failed-pipeline-slack-message.rb b/scripts/generate-failed-pipeline-slack-message.rb
index b695cdfdbee..eefdebd5db5 100755
--- a/scripts/generate-failed-pipeline-slack-message.rb
+++ b/scripts/generate-failed-pipeline-slack-message.rb
@@ -107,7 +107,7 @@ class GenerateFailedPipelineSlackMessage
     if incident_exist?
       incident['web_url']
     else
-      "#{ENV['CI_SERVER_URL']}/#{ENV['BROKEN_MASTER_INCIDENTS_PROJECT']}/-/issues/new?" \
+      "#{ENV['CI_SERVER_URL']}/#{ENV['BROKEN_BRANCH_INCIDENTS_PROJECT']}/-/issues/new?" \
         "issuable_template=incident&issue%5Bissue_type%5D=incident"
     end
   end
diff --git a/spec/scripts/api/commit_merge_requests_spec.rb b/spec/scripts/api/commit_merge_requests_spec.rb
new file mode 100644
index 00000000000..461e3e2e068
--- /dev/null
+++ b/spec/scripts/api/commit_merge_requests_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+require_relative '../../../scripts/api/commit_merge_requests'
+
+RSpec.describe CommitMergeRequests, feature_category: :tooling do
+  describe '#execute' do
+    let(:options) do
+      {
+        sha: 'asdf1234',
+        api_token: 'token',
+        project: 12345,
+        endpoint: 'https://example.gitlab.com'
+      }
+    end
+
+    subject { described_class.new(options).execute }
+
+    it 'requests commit_merge_requests from the gitlab client' do
+      expected_result = ['results']
+      client = double('Gitlab::Client', commit_merge_requests: expected_result) # rubocop:disable RSpec/VerifiedDoubles
+
+      expect(Gitlab).to receive(:client)
+        .with(endpoint: options[:endpoint], private_token: options[:api_token])
+        .and_return(client)
+
+      expect(subject).to eq(expected_result)
+    end
+  end
+end
diff --git a/spec/scripts/create_pipeline_failure_incident_spec.rb b/spec/scripts/create_pipeline_failure_incident_spec.rb
new file mode 100644
index 00000000000..8549cec1b12
--- /dev/null
+++ b/spec/scripts/create_pipeline_failure_incident_spec.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+require_relative '../../scripts/create-pipeline-failure-incident'
+require_relative '../support/helpers/stub_env'
+
+RSpec.describe CreatePipelineFailureIncident, feature_category: :tooling do
+  include StubENV
+
+  describe '#execute' do
+    let(:create_issue) { instance_double(CreateIssue) }
+    let(:issue) { double('Issue', iid: 1) } # rubocop:disable RSpec/VerifiedDoubles
+    let(:create_issue_discussion) { instance_double(CreateIssueDiscussion, execute: true) }
+    let(:failed_jobs) { instance_double(PipelineFailedJobs, execute: []) }
+
+    let(:options) do
+      {
+        project: 1234,
+        api_token: 'asdf1234'
+      }
+    end
+
+    let(:issue_params) do
+      {
+        issue_type: 'incident',
+        title: title,
+        description: description,
+        labels: incident_labels
+      }
+    end
+
+    subject { described_class.new(options).execute }
+
+    before do
+      stub_env(
+        'CI_COMMIT_SHA' => 'bfcd2b9b5cad0b889494ce830697392c8ca11257',
+        'CI_PROJECT_PATH' => 'gitlab.com/gitlab-org/gitlab',
+        'CI_PROJECT_NAME' => 'gitlab',
+        'GITLAB_USER_ID' => '1111',
+        'CI_PROJECT_ID' => '13083',
+        'CI_PIPELINE_ID' => '1234567',
+        'CI_PIPELINE_URL' => 'https://gitlab.com/gitlab-org/gitlab/-/pipelines/1234567',
+        'CI_PROJECT_URL' => 'https://gitlab.com/gitlab-org/gitlab',
+        'CI_PIPELINE_CREATED_AT' => '2023-01-24 00:00:00',
+        'CI_COMMIT_TITLE' => 'Commit title',
+        'CI_PIPELINE_SOURCE' => 'push',
+        'GITLAB_USER_NAME' => 'Foo User',
+        'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE' => 'asdf1234',
+        'CI_SERVER_URL' => 'https://gitlab.com',
+        'GITLAB_USER_LOGIN' => 'foo'
+      )
+    end
+
+    shared_examples 'creating an issue' do
+      it 'successfully creates an issue' do
+        allow(PipelineFailedJobs).to receive(:new)
+          .with(API::DEFAULT_OPTIONS.merge(exclude_allowed_to_fail_jobs: true))
+          .and_return(failed_jobs)
+
+        expect(CreateIssue).to receive(:new)
+          .with(project: options[:project], api_token: options[:api_token])
+          .and_return(create_issue)
+
+        expect(CreateIssueDiscussion).to receive(:new)
+          .with(project: options[:project], api_token: options[:api_token])
+          .and_return(create_issue_discussion).twice
+
+        expect(create_issue).to receive(:execute)
+          .with(issue_params).and_return(issue)
+
+        expect(subject).to eq(issue)
+      end
+    end
+
+    context 'when stable branch' do
+      let(:incident_labels) { ['release-blocker'] }
+      let(:title) { /broken `15-6-stable-ee`/ }
+      let(:description) { /A broken stable branch prevents patch releases/ }
+
+      let(:commit_merge_request) do
+        {
+          'author' => {
+            'id' => '2'
+          },
+          'title' => 'foo',
+          'web_url' => 'https://gitlab.com/test'
+        }
+      end
+
+      let(:merge_request) { instance_double(CommitMergeRequests, execute: [commit_merge_request]) }
+      let(:issue_params) { super().merge(assignee_ids: [1111, 2]) }
+
+      before do
+        stub_env(
+          'CI_COMMIT_REF_NAME' => '15-6-stable-ee'
+        )
+
+        allow(CommitMergeRequests).to receive(:new)
+          .with(API::DEFAULT_OPTIONS.merge(sha: ENV['CI_COMMIT_SHA']))
+          .and_return(merge_request)
+      end
+
+      it_behaves_like 'creating an issue'
+    end
+
+    context 'when other branch' do
+      let(:incident_labels) { ['Engineering Productivity', 'master-broken::undetermined', 'master:broken'] }
+      let(:title) { /broken `master`/ }
+      let(:description) { /Follow the \[Broken `master` handbook guide\]/ }
+
+      before do
+        stub_env(
+          'CI_COMMIT_REF_NAME' => 'master'
+        )
+      end
+
+      it_behaves_like 'creating an issue'
+    end
+  end
+end