diff options
Diffstat (limited to 'doc/development/pipelines.md')
-rw-r--r-- | doc/development/pipelines.md | 134 |
1 files changed, 90 insertions, 44 deletions
diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index aef14535a96..7756ef376fc 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -56,7 +56,7 @@ graph LR subgraph "No needed jobs"; 1-1["danger-review (3.5 minutes)"]; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" - 1-50["docs lint (6.75 minutes)"]; + 1-50["docs lint (9 minutes)"]; click 1-50 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356757&udv=0" end ``` @@ -72,21 +72,21 @@ graph RL; subgraph "No needed jobs"; 1-1["danger-review (3.5 minutes)"]; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" - 1-2["build-qa-image (3.4 minutes)"]; + 1-2["build-qa-image (2.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-test-assets (9.06 minutes)"]; + 1-3["compile-test-assets (8.5 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["compile-production-assets (22 minutes)"]; + 1-5["compile-production-assets (19 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" - 1-6["setup-test-env (8.22 minutes)"]; + 1-6["setup-test-env (7.4 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" 1-7["review-stop-failed-deployment"]; 1-8["dependency_scanning"]; 1-9["qa:internal, qa:internal-as-if-foss"]; 1-11["qa:selectors, qa:selectors-as-if-foss"]; - 1-14["retrieve-tests-metadata (1.5 minutes)"]; + 1-14["retrieve-tests-metadata (1.9 minutes)"]; click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" 1-15["code_quality"]; 1-16["brakeman-sast"]; @@ -113,11 +113,9 @@ graph RL; 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; end - 2_2-2["frontend-fixtures (17.2 minutes)"]; + 2_2-2["frontend-fixtures (16.5 minutes)"]; class 2_2-2 criticalPath; click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" - 2_2-3["frontend-fixtures-as-if-foss (8.75 minutes)"]; - click 2_2-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910154&udv=0" 2_2-4["memory-on-boot (7.19 minutes)"]; click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" 2_2-5["webpack-dev-server (6.1 minutes)"]; @@ -148,21 +146,15 @@ graph RL; 3_1-1["jest (15 minutes)"]; class 3_1-1 criticalPath; click 3_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914204&udv=0" - 3_1-2["karma (8 minutes)"]; + 3_1-2["karma (4 minutes)"]; click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914200&udv=0" - 3_1-3["jest-as-if-foss (19.7 minutes)"]; - click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914205&udv=0" - 3_1-4["karma-as-if-foss (7.5 minutes)"]; - click 3_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914203&udv=0" subgraph "Needs `frontend-fixtures`"; 3_1-1 & 3_1-2 --> 2_2-2; - 3_1-3 & 3_1-4 --> 2_2-3; end - 3_2-1["rspec:coverage (6.5 minutes)"]; + 3_2-1["rspec:coverage (7.5 minutes)"]; subgraph "Depends on `rspec` jobs"; 3_2-1 -.->|"(don't use needs because of limitations)"| 2_5-1; - class 3_2-1 criticalPath; click 3_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7248745&udv=0" end @@ -185,21 +177,21 @@ graph RL; subgraph "No needed jobs"; 1-1["danger-review (3.5 minutes)"]; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" - 1-2["build-qa-image (3.4 minutes)"]; + 1-2["build-qa-image (2.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-test-assets (9.06 minutes)"]; + 1-3["compile-test-assets (8.5 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["compile-production-assets (22 minutes)"]; + 1-5["compile-production-assets (19 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" - 1-6["setup-test-env (8.22 minutes)"]; + 1-6["setup-test-env (7.4 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" 1-7["review-stop-failed-deployment"]; 1-8["dependency_scanning"]; 1-9["qa:internal, qa:internal-as-if-foss"]; 1-11["qa:selectors, qa:selectors-as-if-foss"]; - 1-14["retrieve-tests-metadata (1.5 minutes)"]; + 1-14["retrieve-tests-metadata (1.9 minutes)"]; click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" 1-15["code_quality"]; 1-16["brakeman-sast"]; @@ -227,11 +219,9 @@ graph RL; 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; end - 2_2-2["frontend-fixtures (17.2 minutes)"]; + 2_2-2["frontend-fixtures (16.5 minutes)"]; class 2_2-2 criticalPath; click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" - 2_2-3["frontend-fixtures-as-if-foss (8.75 minutes)"]; - click 2_2-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910154&udv=0" 2_2-4["memory-on-boot (7.19 minutes)"]; click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" 2_2-5["webpack-dev-server (6.1 minutes)"]; @@ -270,21 +260,15 @@ graph RL; 3_1-1["jest (15 minutes)"]; class 3_1-1 criticalPath; click 3_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914204&udv=0" - 3_1-2["karma (8 minutes)"]; + 3_1-2["karma (4 minutes)"]; click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914200&udv=0" - 3_1-3["jest-as-if-foss (19.7 minutes)"]; - click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914205&udv=0" - 3_1-4["karma-as-if-foss (7.5 minutes)"]; - click 3_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914203&udv=0" subgraph "Needs `frontend-fixtures`"; - 3_1-1 & 3_1-3 --> 2_2-2; - 3_1-2 & 3_1-4 --> 2_2-3; + 3_1-1 & 3_1-2 --> 2_2-2; end - 3_2-1["rspec:coverage (6.5 minutes)"]; + 3_2-1["rspec:coverage (7.5 minutes)"]; subgraph "Depends on `rspec` jobs"; 3_2-1 -.->|"(don't use needs because of limitations)"| 2_5-1; - class 3_2-1 criticalPath; click 3_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7248745&udv=0" end @@ -325,21 +309,21 @@ graph RL; subgraph "No needed jobs"; 1-1["danger-review (3.5 minutes)"]; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" - 1-2["build-qa-image (3.4 minutes)"]; + 1-2["build-qa-image (2.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-test-assets (9.06 minutes)"]; + 1-3["compile-test-assets (8.5 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["compile-production-assets (22 minutes)"]; + 1-5["compile-production-assets (19 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" - 1-6["setup-test-env (8.22 minutes)"]; + 1-6["setup-test-env (7.4 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" 1-7["review-stop-failed-deployment"]; 1-8["dependency_scanning"]; 1-9["qa:internal, qa:internal-as-if-foss"]; 1-11["qa:selectors, qa:selectors-as-if-foss"]; - 1-14["retrieve-tests-metadata (1.5 minutes)"]; + 1-14["retrieve-tests-metadata (1.9 minutes)"]; click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" 1-15["code_quality"]; 1-16["brakeman-sast"]; @@ -373,6 +357,65 @@ graph RL; end ``` +### Fail-fast pipeline in Merge Requests + +To provide faster feedback when a Merge Request breaks existing tests, we are experimenting with a +fail-fast mechanism. + +An `rspec fail-fast` job is added in parallel to all other `rspec` jobs in a Merge +Request pipeline. This job runs the tests that are directly related to the changes +in the Merge Request. + +If any of these tests fail, the `rspec fail-fast` job fails, triggering a +`fail-pipeline-early` job to run. The `fail-pipeline-early` job: + +- Cancels the currently running pipeline and all in-progress jobs. +- Sets pipeline to have status `failed`. + +For example: + +```mermaid +graph LR + subgraph "prepare stage"; + A["detect-tests"] + end + + subgraph "test stage"; + B["jest"]; + C["rspec migration"]; + D["rspec unit"]; + E["rspec integration"]; + F["rspec system"]; + G["rspec fail-fast"]; + end + + subgraph "post-test stage"; + Z["fail-pipeline-early"]; + end + + A --"artifact: list of test files"--> G + G --"on failure"--> Z +``` + +A Merge Request author may choose to opt-out of the fail fast mechanism by doing one of the following: + +- Including `[SKIP RSPEC FAIL-FAST]` in the Merge Request title. +- Starting the `dont-interrupt-me` job found in the `sync` stage of a Merge Request pipeline. + +The `rspec fail-fast` is a no-op if there are more than 10 test files related to the +Merge Request. This prevents `rspec fail-fast` duration from exceeding the average +`rspec` job duration and defeating its purpose. + +This number can be overridden by setting a CI variable named `RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD`. + +NOTE: **Note:** +This experiment is only enabled when the CI variable `RSPEC_FAIL_FAST_ENABLED=true` is set. + +#### Determining related test files in a Merge Request + +The test files related to the Merge Request are determined using the [`test_file_finder`](https://gitlab.com/gitlab-org/ci-cd/test_file_finder) gem. +We are using a custom mapping between source file to test files, maintained in the `tests.yml` file. + ### PostgreSQL versions testing #### Current versions testing @@ -417,8 +460,8 @@ of the `gitlab-org/gitlab-foss` project. These jobs are only created in the foll - Merge requests which include `RUN AS-IF-FOSS` in their title. - Merge requests that changes the CI configuration. -The `* as-if-foss` jobs have the `FOSS_ONLY='1'` variable set and gets their EE-specific -folders removed before the tests start running. +The `* as-if-foss` jobs are run in addition to the regular EE-context jobs. They have the `FOSS_ONLY='1'` variable +set and get their EE-specific folders removed before the tests start running. The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to the `gitlab-org/gitlab-foss` project. @@ -514,8 +557,9 @@ overwrites the Git configuration with the appropriate settings to fetch from the GitLab repository. `CI_REPO_CACHE_CREDENTIALS` contains the Google Cloud service account -JSON for uploading to the `gitlab-ci-git-repo-cache` bucket. These -credentials are stored in the 1Password GitLab.com Production vault. +JSON for uploading to the `gitlab-ci-git-repo-cache` bucket. (If you’re a +GitLab Team Member, find credentials in the +[GitLab shared 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams). Note that this bucket should be located in the same continent as the runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing). @@ -540,8 +584,10 @@ The current stages are: later used by the (Helm) Review App deployment (see [Review Apps](testing_guide/review_apps.md) for details). - `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps. +- `dast`: This stage includes jobs that run a DAST full scan against the Review App +that is deployed in stage `review`. - `qa`: This stage includes jobs that perform QA tasks against the Review App - that is deployed in the previous stage. + that is deployed in stage `review`. - `post-qa`: This stage includes jobs that build reports or gather data from the `qa` stage's jobs (e.g. Review App performance report). - `pages`: This stage includes a job that deploys the various reports as |