diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-07 00:09:33 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-07 00:09:33 +0000 |
| commit | b56027c9d80ac0e297ba8a43c81e8504172dbf9f (patch) | |
| tree | b85f743277145e930ae195664655d696e6e0a7fc | |
| parent | 7915c41e4261719719e791602c8235574157164c (diff) | |
| download | gitlab-ce-b56027c9d80ac0e297ba8a43c81e8504172dbf9f.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
47 files changed, 859 insertions, 670 deletions
diff --git a/app/controllers/import/gitea_controller.rb b/app/controllers/import/gitea_controller.rb index f0888e08622..42c23fb29a7 100644 --- a/app/controllers/import/gitea_controller.rb +++ b/app/controllers/import/gitea_controller.rb @@ -3,6 +3,8 @@ class Import::GiteaController < Import::GithubController extend ::Gitlab::Utils::Override + before_action :verify_blocked_uri, only: :status + def new if session[access_token_key].present? && provider_url.present? redirect_to status_import_url @@ -16,13 +18,7 @@ class Import::GiteaController < Import::GithubController # Must be defined or it will 404 def status - if blocked_url? - session[access_token_key] = nil - - redirect_to new_import_url, alert: _('Specified URL cannot be used.') - else - super - end + super end private @@ -61,8 +57,8 @@ class Import::GiteaController < Import::GithubController { host: provider_url, api_version: 'v1' } end - def blocked_url? - Gitlab::UrlBlocker.blocked_url?( + def verify_blocked_uri + Gitlab::UrlBlocker.validate!( provider_url, { allow_localhost: allow_local_requests?, @@ -70,6 +66,10 @@ class Import::GiteaController < Import::GithubController schemes: %w(http https) } ) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + session[access_token_key] = nil + + redirect_to new_import_url, alert: _('Specified URL cannot be used: "%{reason}"') % { reason: e.message } end def allow_local_requests? diff --git a/app/controllers/projects/import/jira_controller.rb b/app/controllers/projects/import/jira_controller.rb index 03ae17e18be..6af630a9528 100644 --- a/app/controllers/projects/import/jira_controller.rb +++ b/app/controllers/projects/import/jira_controller.rb @@ -22,8 +22,14 @@ module Projects end def import - response = ::JiraImport::StartImportService.new(current_user, @project, jira_import_params[:jira_project_key]).execute - flash[:notice] = response.message if response.message.present? + jira_project_key = jira_import_params[:jira_project_key] + + if jira_project_key.present? + response = ::JiraImport::StartImportService.new(current_user, @project, jira_project_key).execute + flash[:notice] = response.message if response.message.present? + else + flash[:alert] = 'No jira project key has been provided.' + end redirect_to project_import_jira_path(@project) end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 973bdc54111..cfe1c77ec48 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -262,6 +262,15 @@ class Snippet < ApplicationRecord @repository ||= Repository.new(full_path, self, shard: repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::SNIPPET) end + def repository_size_checker + strong_memoize(:repository_size_checker) do + ::Gitlab::RepositorySizeChecker.new( + current_size_proc: -> { repository._uncached_size.megabytes }, + limit: Gitlab::CurrentSettings.snippet_size_limit + ) + end + end + def storage @storage ||= Storage::Hashed.new(self, prefix: Storage::Hashed::SNIPPET_REPOSITORY_PATH_PREFIX) end diff --git a/changelogs/unreleased/normalize-import-error-messages.yml b/changelogs/unreleased/normalize-import-error-messages.yml new file mode 100644 index 00000000000..13b3d84448b --- /dev/null +++ b/changelogs/unreleased/normalize-import-error-messages.yml @@ -0,0 +1,5 @@ +--- +title: Normalize error message between Gitea and Fogbugz importers +merge_request: 28802 +author: +type: other diff --git a/doc/README.md b/doc/README.md index 8c6f76306c4..bcb3140887d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -279,7 +279,7 @@ The following documentation relates to the DevOps **Release** stage: | Release Topics | Description | |:------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------| -| [Auto Deploy](topics/autodevops/index.md#auto-deploy) | Configure GitLab for the deployment of your application. | +| [Auto Deploy](topics/autodevops/stages.md#auto-deploy) | Configure GitLab for the deployment of your application. | | [Canary Deployments](user/project/canary_deployments.md) **(PREMIUM)** | Employ a popular CI strategy where a small portion of the fleet is updated to the new version first. | | [Deploy Boards](user/project/deploy_boards.md) **(PREMIUM)** | View the current health and status of each CI environment running on Kubernetes, displaying the status of the pods in the deployment. | | [Environments and deployments](ci/environments.md) | With environments, you can control the continuous deployment of your software within GitLab. | diff --git a/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png b/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png Binary files differindex 933d8fb6f36..807936423e5 100644 --- a/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png +++ b/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png diff --git a/doc/ci/README.md b/doc/ci/README.md index f902400efc2..30971d422cc 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -120,7 +120,7 @@ Its feature set is listed on the table below according to DevOps stages. | [Using Docker images](docker/using_docker_images.md) | Use GitLab and GitLab Runner with Docker to build and test applications. | |---+---| | **Release** || -| [Auto Deploy](../topics/autodevops/index.md#auto-deploy) | Deploy your application to a production environment in a Kubernetes cluster. | +| [Auto Deploy](../topics/autodevops/stages.md#auto-deploy) | Deploy your application to a production environment in a Kubernetes cluster. | | [Building Docker images](docker/using_docker_build.md) | Maintain Docker-based projects using GitLab CI/CD. | | [Canary Deployments](../user/project/canary_deployments.md) **(PREMIUM)** | Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature. | | [Deploy Boards](../user/project/deploy_boards.md) **(PREMIUM)** | Check the current health and status of each CI/CD environment running on Kubernetes. | diff --git a/doc/ci/autodeploy/index.md b/doc/ci/autodeploy/index.md index 5221cbf8609..ca2df72e32e 100644 --- a/doc/ci/autodeploy/index.md +++ b/doc/ci/autodeploy/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../topics/autodevops/index.md#auto-deploy' +redirect_to: '../../topics/autodevops/stages.md#auto-deploy' --- -This document was moved to [another location](../../topics/autodevops/index.md#auto-deploy). +This document was moved to [another location](../../topics/autodevops/stages.md#auto-deploy). diff --git a/doc/ci/autodeploy/quick_start_guide.md b/doc/ci/autodeploy/quick_start_guide.md index 5221cbf8609..ca2df72e32e 100644 --- a/doc/ci/autodeploy/quick_start_guide.md +++ b/doc/ci/autodeploy/quick_start_guide.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../topics/autodevops/index.md#auto-deploy' +redirect_to: '../../topics/autodevops/stages.md#auto-deploy' --- -This document was moved to [another location](../../topics/autodevops/index.md#auto-deploy). +This document was moved to [another location](../../topics/autodevops/stages.md#auto-deploy). diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index 069430579e3..b16cde54b93 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -199,7 +199,7 @@ according to each stage (Verify, Package, Release). - Deploy your features behind [Feature Flags](../../user/project/operations/feature_flags.md). **(PREMIUM)** - Add release notes to any Git tag with [GitLab Releases](../../user/project/releases/index.md). - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](../../user/project/deploy_boards.md). **(PREMIUM)** - - Deploy your application to a production environment in a Kubernetes cluster with [Auto Deploy](../../topics/autodevops/index.md#auto-deploy). + - Deploy your application to a production environment in a Kubernetes cluster with [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy). With GitLab CI/CD you can also: diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index 656fb2ed1ad..dba68974dd5 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -18,12 +18,12 @@ which then defines jobs. Some jobs use images that are built from external projects: -- [Auto Build](../topics/autodevops/index.md#auto-build) uses +- [Auto Build](../topics/autodevops/stages.md#auto-build) uses [configuration](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml) in which the `build` job uses an image that is built using the [`auto-build-image`](https://gitlab.com/gitlab-org/cluster-integration/auto-build-image) project. -- [Auto Deploy](../topics/autodevops/index.md#auto-deploy) uses +- [Auto Deploy](../topics/autodevops/stages.md#auto-deploy) uses [configuration](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) in which the jobs defined in this template use an image that is built using the [`auto-deploy-image`](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) diff --git a/doc/development/database_review.md b/doc/development/database_review.md index d95fd7a38f6..f2db0ab4fd5 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -77,7 +77,9 @@ the following preparations into account. - Ensure `db/structure.sql` is updated. - Make migrations reversible by using the `change` method or include a `down` method when using `up`. - Include either a rollback procedure or describe how to rollback changes. -- Add the output of the migration(s) to the MR description. +- Add the output of both migrating and rolling back for all migrations into the MR description + - Ensure the down method reverts the changes in `db/structure.sql` + - Update the migration output whenever you modify the migrations during the review process - Add tests for the migration in `spec/migrations` if necessary. See [Testing Rails migrations at GitLab](testing_guide/testing_migrations_guide.md) for more details. - When [high-traffic](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L12) tables are involved in the migration, use the [`with_lock_retries`](migration_style_guide.md#retry-mechanism-when-acquiring-database-locks) helper method. Review the relevant [examples in our documentation](migration_style_guide.md#examples) for use cases and solutions. - Ensure RuboCop checks are not disabled unless there's a valid reason to. diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index e7a43f769b3..84ba47eba78 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -21,8 +21,10 @@ with exceptions and details noted below and in the template included on this pag - **Title**: Top-level heading with the feature name, or a use case name, which would start with a verb, like Configuring, Enabling, etc. -- **Introduction**: A couple sentences about the subject matter and what's to be found on this page. -- **Overview** Describe what it is, what it does, and in what context it should be used. +- **Introduction**: A couple sentences about the subject matter and what's to be found +on this page. Describe what the feature or topic is, what it does, and in what context it should +be used. There is no need to add a title called "Introduction" or "Overview," because people rarely + search for these terms. Just put this information after the title. - **Use cases**: describes real use case scenarios for that feature/configuration. - **Requirements**: describes what software, configuration, account, or knowledge is required. - **Instructions**: One or more sets of detailed instructions to follow. @@ -54,12 +56,10 @@ description: "Short document description." # Up to ~200 chars long. They will be > [Introduced](link_to_issue_or_mr) in GitLab (Tier) X.Y (2). An introduction -- without its own additional header -- goes here. -Offer a very short description of the feature or use case, and what to expect on this page. +Offer a description of the feature or use case, and what to expect on this page. (You can reuse this content, or part of it, for the front matter's `description` at the top of this file). -## Overview - -The feature overview should answer the following questions: +The introduction should answer the following questions: - What is this feature or use case? - Who is it for? diff --git a/doc/topics/application_development_platform/index.md b/doc/topics/application_development_platform/index.md index 8de440c7f00..df585aae7e8 100644 --- a/doc/topics/application_development_platform/index.md +++ b/doc/topics/application_development_platform/index.md @@ -47,10 +47,10 @@ that may lead to security problems and unintended use. This can be achieved by m which inform security teams and developers if there is something to consider changing in their apps before it is too late to create a preventative fix. The following features are included: -- [Auto SAST (Static Application Security Testing)](../autodevops/index.md#auto-sast-ultimate) -- [Auto Dependency Scanning](../autodevops/index.md#auto-dependency-scanning-ultimate) -- [Auto Container Scanning](../autodevops/index.md#auto-container-scanning-ultimate) -- [Auto DAST (Dynamic Application Security Testing)](../autodevops/index.md#auto-dast-ultimate) +- [Auto SAST (Static Application Security Testing)](../autodevops/stages.md#auto-sast-ultimate) +- [Auto Dependency Scanning](../autodevops/stages.md#auto-dependency-scanning-ultimate) +- [Auto Container Scanning](../autodevops/stages.md#auto-container-scanning-ultimate) +- [Auto DAST (Dynamic Application Security Testing)](../autodevops/stages.md#auto-dast-ultimate) ### Observability @@ -58,5 +58,5 @@ Performance is a critical aspect of the user experience, and ensuring your appli responsibility. The Application Development Platform integrates key performance analytics and feedback into GitLab, automatically. The following features are included: -- [Auto Monitoring](../autodevops/index.md#auto-monitoring) +- [Auto Monitoring](../autodevops/stages.md#auto-monitoring) - [In-app Kubernetes Logs](../../user/project/clusters/kubernetes_pod_logs.md) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 087b6745a66..b8b16096c28 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -28,7 +28,7 @@ It will be automatically disabled in individual projects on their first pipeline if it has not been explicitly enabled for the project. Since [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/issues/26655), Auto DevOps -will run on pipelines automatically only if a [`Dockerfile` or matching buildpack](#auto-build) +will run on pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists. If a [CI/CD configuration file](../../ci/yaml/README.md) is present in the project, @@ -68,18 +68,18 @@ in multiple ways: Comprised of a set of stages, Auto DevOps brings these best practices to your project in a simple and automatic way: -1. [Auto Build](#auto-build) -1. [Auto Test](#auto-test) -1. [Auto Code Quality](#auto-code-quality-starter) **(STARTER)** -1. [Auto SAST (Static Application Security Testing)](#auto-sast-ultimate) **(ULTIMATE)** -1. [Auto Dependency Scanning](#auto-dependency-scanning-ultimate) **(ULTIMATE)** -1. [Auto License Compliance](#auto-license-compliance-ultimate) **(ULTIMATE)** -1. [Auto Container Scanning](#auto-container-scanning-ultimate) **(ULTIMATE)** -1. [Auto Review Apps](#auto-review-apps) -1. [Auto DAST (Dynamic Application Security Testing)](#auto-dast-ultimate) **(ULTIMATE)** -1. [Auto Deploy](#auto-deploy) -1. [Auto Browser Performance Testing](#auto-browser-performance-testing-premium) **(PREMIUM)** -1. [Auto Monitoring](#auto-monitoring) +1. [Auto Build](stages.md#auto-build) +1. [Auto Test](stages.md#auto-test) +1. [Auto Code Quality](stages.md#auto-code-quality-starter) **(STARTER)** +1. [Auto SAST (Static Application Security Testing)](stages.md#auto-sast-ultimate) **(ULTIMATE)** +1. [Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate) **(ULTIMATE)** +1. [Auto License Compliance](stages.md#auto-license-compliance-ultimate) **(ULTIMATE)** +1. [Auto Container Scanning](stages.md#auto-container-scanning-ultimate) **(ULTIMATE)** +1. [Auto Review Apps](stages.md#auto-review-apps) +1. [Auto DAST (Dynamic Application Security Testing)](stages.md#auto-dast-ultimate) **(ULTIMATE)** +1. [Auto Deploy](stages.md#auto-deploy) +1. [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium) **(PREMIUM)** +1. [Auto Monitoring](stages.md#auto-monitoring) As Auto DevOps relies on many different components, it's good to have a basic knowledge of the following: @@ -110,7 +110,7 @@ To make full use of Auto DevOps, you will need: 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for the project. The easiest way is to create a [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). - For Kubernetes 1.16+ clusters, there is some additional configuration for [Auto Deploy for Kubernetes 1.16+](#kubernetes-116). + For Kubernetes 1.16+ clusters, there is some additional configuration for [Auto Deploy for Kubernetes 1.16+](stages.md#kubernetes-116). 1. NGINX Ingress. You can deploy it to your Kubernetes cluster by installing the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), once you have configured GitLab's Kubernetes integration in the previous step. @@ -191,8 +191,8 @@ One all requirements are met, you can go ahead and [enable Auto DevOps](#enablin ## Auto DevOps base domain The Auto DevOps base domain is required if you want to make use of -[Auto Review Apps](#auto-review-apps), [Auto Deploy](#auto-deploy), and -[Auto Monitoring](#auto-monitoring). It can be defined in any of the following +[Auto Review Apps](stages.md#auto-review-apps), [Auto Deploy](stages.md#auto-deploy), and +[Auto Monitoring](stages.md#auto-monitoring). It can be defined in any of the following places: - either under the cluster's settings, whether for [projects](../../user/project/clusters/index.md#base-domain) or [groups](../../user/group/clusters/index.md#base-domain) @@ -248,7 +248,7 @@ If enabling, check that your project doesn't have a `.gitlab-ci.yml`, or if one 1. Go to your project's **Settings > CI/CD > Auto DevOps**. 1. Toggle the **Default to Auto DevOps pipeline** checkbox (checked to enable, unchecked to disable) 1. When enabling, it's optional but recommended to add in the [base domain](#auto-devops-base-domain) - that will be used by Auto DevOps to [deploy your application](#auto-deploy) + that will be used by Auto DevOps to [deploy your application](stages.md#auto-deploy) and choose the [deployment strategy](#deployment-strategy). 1. Click **Save changes** for the changes to take effect. @@ -297,7 +297,7 @@ project's **Settings > CI/CD > Auto DevOps**. The available options are: -- **Continuous deployment to production**: Enables [Auto Deploy](#auto-deploy) +- **Continuous deployment to production**: Enables [Auto Deploy](stages.md#auto-deploy) with `master` branch directly deployed to production. - **Continuous deployment to production using timed incremental rollout**: Sets the [`INCREMENTAL_ROLLOUT_MODE`](#timed-incremental-rollout-to-production-premium) variable @@ -324,7 +324,7 @@ are currently 3 defined environment names that you need to know: - `staging` - `production` -Those environments are tied to jobs that use [Auto Deploy](#auto-deploy), so +Those environments are tied to jobs that use [Auto Deploy](stages.md#auto-deploy), so except for the environment scope, they would also need to have a different domain they would be deployed to. This is why you need to define a separate `KUBE_INGRESS_BASE_DOMAIN` variable for all the above @@ -358,553 +358,6 @@ and verifying that your app is deployed as a review app in the Kubernetes cluster with the `review/*` environment scope. Similarly, you can check the other environments. -## Stages of Auto DevOps - -The following sections describe the stages of Auto DevOps. Read them carefully -to understand how each one works. - -### Auto Build - -Auto Build creates a build of the application using an existing `Dockerfile` or -Heroku buildpacks. - -Either way, the resulting Docker image is automatically pushed to the -[Container Registry](../../user/packages/container_registry/index.md) and tagged with the commit SHA or tag. - -#### Auto Build using a Dockerfile - -If a project's repository contains a `Dockerfile` at its root, Auto Build will use -`docker build` to create a Docker image. - -If you are also using Auto Review Apps and Auto Deploy and choose to provide -your own `Dockerfile`, make sure you expose your application to port -`5000` as this is the port assumed by the -[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](#custom-helm-chart) - -#### Auto Build using Heroku buildpacks - -Auto Build builds an application using a project's `Dockerfile` if present, or -otherwise it will use [Herokuish](https://github.com/gliderlabs/herokuish) -and [Heroku buildpacks](https://devcenter.heroku.com/articles/buildpacks) -to automatically detect and build the application into a Docker image. - -Each buildpack requires certain files to be in your project's repository for -Auto Build to successfully build your application. For example, the following -files are required at the root of your application's repository, depending on -the language: - -- A `Pipfile` or `requirements.txt` file for Python projects. -- A `Gemfile` or `Gemfile.lock` file for Ruby projects. - -For the requirements of other languages and frameworks, read the -[buildpacks docs](https://devcenter.heroku.com/articles/buildpacks#officially-supported-buildpacks). - -TIP: **Tip:** -If Auto Build fails despite the project meeting the buildpack requirements, set -a project variable `TRACE=true` to enable verbose logging, which may help to -troubleshoot. - -#### Auto Build using Cloud Native Buildpacks (beta) - -> Introduced in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28165). - -Auto Build supports building your application using [Cloud Native Buildpacks](https://buildpacks.io) -through the [`pack` command](https://github.com/buildpacks/pack). To use Cloud Native Buildpacks, -set the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` to a non-empty value. - -Cloud Native Buildpacks (CNBs) are an evolution of Heroku buildpacks, and -will eventually supersede Herokuish-based builds within Auto DevOps. For more -information, see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212692). - -Builds using Cloud Native Buildpacks support the same options as builds using -Heroku buildpacks, with the following caveats: - -- The buildpack must be a Cloud Native Buildpack. A Heroku buildpack can be - converted to a Cloud Native Buildpack using Heroku's - [`cnb-shim`](https://github.com/heroku/cnb-shim). -- `BUILDPACK_URL` must be in a form - [supported by `pack`](https://buildpacks.io/docs/app-developer-guide/specific-buildpacks/). -- The `/bin/herokuish` command is not present in the resulting image, and prefixing - commands with `/bin/herokuish procfile exec` is no longer required (nor possible). - -NOTE: **Note**: Auto Test still uses Herokuish, as test suite detection is not -yet part of the Cloud Native Buildpack specification. For more information, see -[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212689). - -### Auto Test - -Auto Test automatically runs the appropriate tests for your application using -[Herokuish](https://github.com/gliderlabs/herokuish) and [Heroku -buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing -your project to detect the language and framework. Several languages and -frameworks are detected automatically, but if your language is not detected, -you may succeed with a [custom buildpack](#custom-buildpacks). Check the -[currently supported languages](#currently-supported-languages). - -Auto Test uses tests you already have in your application. If there are no -tests, it's up to you to add them. - -### Auto Code Quality **(STARTER)** - -Auto Code Quality uses the -[Code Quality image](https://gitlab.com/gitlab-org/ci-cd/codequality) to run -static analysis and other code checks on the current code. The report is -created, and is uploaded as an artifact which you can later download and check -out. - -Any differences between the source and target branches are also -[shown in the merge request widget](../../user/project/merge_requests/code_quality.md). - -### Auto SAST **(ULTIMATE)** - -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. - -Static Application Security Testing (SAST) uses the -[SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static -analysis on the current code and checks for potential security issues. The -Auto SAST stage will be skipped on licenses other than Ultimate and requires GitLab Runner 11.5 or above. - -Once the report is created, it's uploaded as an artifact which you can later download and -check out. - -Any security warnings are also shown in the merge request widget. Read more how -[SAST works](../../user/application_security/sast/index.md). - -### Auto Dependency Scanning **(ULTIMATE)** - -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. - -Dependency Scanning uses the -[Dependency Scanning Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning) -to run analysis on the project dependencies and checks for potential security issues. -The Auto Dependency Scanning stage will be skipped on licenses other than Ultimate -and requires GitLab Runner 11.5 or above. - -Once the -report is created, it's uploaded as an artifact which you can later download and -check out. - -Any security warnings are also shown in the merge request widget. Read more about -[Dependency Scanning](../../user/application_security/dependency_scanning/index.md). - -### Auto License Compliance **(ULTIMATE)** - -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. - -License Compliance uses the -[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) -to search the project dependencies for their license. The Auto License Compliance stage -will be skipped on licenses other than Ultimate. - -Once the -report is created, it's uploaded as an artifact which you can later download and -check out. - -Any licenses are also shown in the merge request widget. Read more how -[License Compliance works](../../user/compliance/license_compliance/index.md). - -### Auto Container Scanning **(ULTIMATE)** - -> Introduced in GitLab 10.4. - -Vulnerability Static Analysis for containers uses -[Clair](https://github.com/quay/clair) to run static analysis on a -Docker image and checks for potential security issues. The Auto Container Scanning stage -will be skipped on licenses other than Ultimate. - -Once the report is -created, it's uploaded as an artifact which you can later download and -check out. - -Any security warnings are also shown in the merge request widget. Read more how -[Container Scanning works](../../user/application_security/container_scanning/index.md). - -### Auto Review Apps - -This is an optional step, since many projects do not have a Kubernetes cluster -available. If the [requirements](#requirements) are not met, the job will -silently be skipped. - -[Review Apps](../../ci/review_apps/index.md) are temporary application environments based on the -branch's code so developers, designers, QA, product managers, and other -reviewers can actually see and interact with code changes as part of the review -process. Auto Review Apps create a Review App for each branch. - -Auto Review Apps will deploy your app to your Kubernetes cluster only. When no cluster -is available, no deployment will occur. - -The Review App will have a unique URL based on the project ID, the branch or tag -name, and a unique number, combined with the Auto DevOps base domain. For -example, `13083-review-project-branch-123456.example.com`. A link to the Review App shows -up in the merge request widget for easy discovery. When the branch or tag is deleted, -for example after the merge request is merged, the Review App will automatically -be deleted. - -Review apps are deployed using the -[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with -Helm, which can be [customized](#custom-helm-chart). The app will be deployed into the [Kubernetes -namespace](../../user/project/clusters/index.md#deployment-variables) -for the environment. - -Since GitLab 11.4, a [local -Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is -used. Previous versions of GitLab had a Tiller installed in the project -namespace. - -CAUTION: **Caution:** -Your apps should *not* be manipulated outside of Helm (using Kubernetes directly). -This can cause confusion with Helm not detecting the change and subsequent -deploys with Auto DevOps can undo your changes. Also, if you change something -and want to undo it by deploying again, Helm may not detect that anything changed -in the first place, and thus not realize that it needs to re-apply the old config. - -### Auto DAST **(ULTIMATE)** - -> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. - -Dynamic Application Security Testing (DAST) uses the -popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) -to perform an analysis on the current code and checks for potential security -issues. The Auto DAST stage will be skipped on licenses other than Ultimate. - -Once the DAST scan is complete, any security warnings are shown -on the [Security Dashboard](../../user/application_security/security_dashboard/index.md) -and the Merge Request Widget. Read how -[DAST works](../../user/application_security/dast/index.md). - -On your default branch, DAST scans an app deployed specifically for that purpose. -The app is deleted after DAST has run. - -On feature branches, DAST scans the [review app](#auto-review-apps). - -#### Overriding the DAST target - -To use a custom target instead of the auto-deployed review apps, -set a `DAST_WEBSITE` environment variable to the URL for DAST to scan. - -NOTE: **Note:** -If [DAST Full Scan](../../user/application_security/dast/index.md#full-scan) is enabled, it is strongly advised **not** -to set `DAST_WEBSITE` to any staging or production environment. DAST Full Scan -actively attacks the target, which can take down the application and lead to -data loss or corruption. - -#### Disabling Auto DAST - -DAST can be disabled: - -- On all branches by setting the `DAST_DISABLED` environment variable to `"true"`. -- Only on the default branch by setting the `DAST_DISABLED_FOR_DEFAULT_BRANCH` environment variable to `"true"`. - -### Auto Browser Performance Testing **(PREMIUM)** - -> Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. - -Auto Browser Performance Testing utilizes the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) to measure the performance of a web page. A JSON report is created and uploaded as an artifact, which includes the overall performance score for each page. By default, the root page of Review and Production environments will be tested. If you would like to add additional URL's to test, simply add the paths to a file named `.gitlab-urls.txt` in the root directory, one per line. For example: - -```text -/ -/features -/direction -``` - -Any performance differences between the source and target branches are also -[shown in the merge request widget](../../user/project/merge_requests/browser_performance_testing.md). - -### Auto Deploy - -This is an optional step, since many projects do not have a Kubernetes cluster -available. If the [requirements](#requirements) are not met, the job will -silently be skipped. - -After a branch or merge request is merged into the project's default branch (usually -`master`), Auto Deploy deploys the application to a `production` environment in -the Kubernetes cluster, with a namespace based on the project name and unique -project ID, for example `project-4321`. - -Auto Deploy doesn't include deployments to staging or canary by default, but the -[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to -enable them. - -You can make use of [environment variables](#environment-variables) to automatically -scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](#custom-helm-chart). - -Apps are deployed using the -[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with -Helm. The app will be deployed into the [Kubernetes -namespace](../../user/project/clusters/index.md#deployment-variables) -for the environment. - -Since GitLab 11.4, a [local -Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is -used. Previous versions of GitLab had a Tiller installed in the project -namespace. - -CAUTION: **Caution:** -Your apps should *not* be manipulated outside of Helm (using Kubernetes directly). -This can cause confusion with Helm not detecting the change and subsequent -deploys with Auto DevOps can undo your changes. Also, if you change something -and want to undo it by deploying again, Helm may not detect that anything changed -in the first place, and thus not realize that it needs to re-apply the old config. - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19507) in GitLab 11.0. - -For internal and private projects a [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token) -will be automatically created, when Auto DevOps is enabled and the Auto DevOps settings are saved. This Deploy Token -can be used for permanent access to the registry. When the GitLab Deploy Token has been manually revoked, it won't be automatically created. - -If the GitLab Deploy Token cannot be found, `CI_REGISTRY_PASSWORD` is -used. Note that `CI_REGISTRY_PASSWORD` is only valid during deployment. -This means that Kubernetes will be able to successfully pull the -container image during deployment but in cases where the image needs to -be pulled again, e.g. after pod eviction, Kubernetes will fail to do so -as it will be attempting to fetch the image using -`CI_REGISTRY_PASSWORD`. - -#### Kubernetes 1.16+ - -> - [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/51) in GitLab 12.8. -> - Support for deploying a PostgreSQL version that supports Kubernetes 1.16+ was [introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/49) in GitLab 12.9. - -CAUTION: **Deprecation** -The default value of `extensions/v1beta1` for the `deploymentApiVersion` setting is -deprecated, and is scheduled to be changed to a new default of `apps/v1` in -[GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/issues/47). - -In Kubernetes 1.16 onwards, a number of [APIs were removed](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/), -including support for `Deployment` in the `extensions/v1beta1` version. - -To use Auto Deploy on a Kubernetes 1.16+ cluster, you must: - -1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](#customize-values-for-helm-chart): - - ```yml - deploymentApiVersion: apps/v1 - ``` - -1. Set the: - - - `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`. - - `POSTGRES_VERSION` variable to `9.6.16` or higher. - - This will opt-in to using a version of the PostgreSQL chart that supports Kubernetes - 1.16 and higher. - -DANGER: **Danger:** Opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version -`2` will delete the version `1` PostgreSQL database. Please follow the -guide on [upgrading PostgreSQL](upgrading_postgresql.md) to backup and -restore your database before opting into version `2`. - -#### Migrations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21955) in GitLab 11.4 - -Database initialization and migrations for PostgreSQL can be configured to run -within the application pod by setting the project variables `DB_INITIALIZE` and -`DB_MIGRATE` respectively. - -If present, `DB_INITIALIZE` will be run as a shell command within an -application pod as a Helm post-install hook. As some applications will -not run without a successful database initialization step, GitLab will -deploy the first release without the application deployment and only the -database initialization step. After the database initialization completes, -GitLab will deploy a second release with the application deployment as -normal. - -Note that a post-install hook means that if any deploy succeeds, -`DB_INITIALIZE` will not be processed thereafter. - -If present, `DB_MIGRATE` will be run as a shell command within an application pod as -a Helm pre-upgrade hook. - -For example, in a Rails application in an image built with -[Herokuish](https://github.com/gliderlabs/herokuish): - -- `DB_INITIALIZE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:setup` -- `DB_MIGRATE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:migrate` - -Unless you have a `Dockerfile` in your repo, your image is built with -Herokuish, and you must prefix commands run in these images with `/bin/herokuish -procfile exec` to replicate the environment where your application will run. - -#### Workers - -Some web applications need to run extra deployments for "worker processes". For -example, it is common in a Rails application to have a separate worker process -to run background tasks like sending emails. - -The [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) -used in Auto Deploy [has support for running worker -processes](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/9). - -In order to run a worker, you'll need to ensure that it is able to respond to -the standard health checks, which expect a successful HTTP response on port -`5000`. For [Sidekiq](https://github.com/mperham/sidekiq), you could make use of -the [`sidekiq_alive` gem](https://rubygems.org/gems/sidekiq_alive) to do this. - -In order to work with Sidekiq, you'll also need to ensure your deployments have -access to a Redis instance. Auto DevOps won't deploy this for you so you'll -need to: - -- Maintain your own Redis instance. -- Set a CI variable `K8S_SECRET_REDIS_URL`, which the URL of this instance to - ensure it's passed into your deployments. - -Once you have configured your worker to respond to health checks, run a Sidekiq -worker for your Rails application. You can enable workers by setting the -following in the [`.gitlab/auto-deploy-values.yaml` file](#customize-values-for-helm-chart): - -```yml -workers: - sidekiq: - replicaCount: 1 - command: - - /bin/herokuish - - procfile - - exec - - sidekiq - preStopCommand: - - /bin/herokuish - - procfile - - exec - - sidekiqctl - - quiet - terminationGracePeriodSeconds: 60 -``` - -#### Network Policy - -> [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/30) in GitLab 12.7. - -By default, all Kubernetes pods are -[non-isolated](https://kubernetes.io/docs/concepts/services-networking/network-policies/#isolated-and-non-isolated-pods), -meaning that they will accept traffic to and from any source. You can use -[NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -to restrict connections to and from selected pods, namespaces, and the Internet. - -NOTE: **Note:** -You must use a Kubernetes network plugin that implements support for -`NetworkPolicy`. The default network plugin for Kubernetes (`kubenet`) -[does not implement](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#kubenet) -support for it. The [Cilium](https://cilium.io/) network plugin can be -installed as a [cluster application](../../user/clusters/applications.md#install-cilium-using-gitlab-cicd) -to enable support for network policies. - -You can enable deployment of a network policy by setting the following -in the `.gitlab/auto-deploy-values.yaml` file: - -```yaml -networkPolicy: - enabled: true -``` - -The default policy deployed by the auto deploy pipeline will allow -traffic within a local namespace and from the `gitlab-managed-apps` -namespace. All other inbound connection will be blocked. Outbound -traffic (for example, to the Internet) is not affected by the default policy. - -You can also provide a custom [policy specification](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#networkpolicyspec-v1-networking-k8s-io) -via the `.gitlab/auto-deploy-values.yaml` file, for example: - -```yaml -networkPolicy: - enabled: true - spec: - podSelector: - matchLabels: - app.gitlab.com/env: staging - ingress: - - from: - - podSelector: - matchLabels: {} - - namespaceSelector: - matchLabels: - app.gitlab.com/managed_by: gitlab -``` - -For more information on how to install Network Policies, see -[Install Cilium using GitLab CI/CD](../../user/clusters/applications.md#install-cilium-using-gitlab-cicd). - -#### Web Application Firewall (ModSecurity) customization - -> [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/44) in GitLab 12.8. - -Customization on an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) or on a deployment base is available for clusters with [ModSecurity installed](../../user/clusters/applications.md#web-application-firewall-modsecurity). - -To enable ModSecurity with Auto Deploy, you need to create a `.gitlab/auto-deploy-values.yaml` file in your project with the following attributes. - -|Attribute | Description | Default | ------------|-------------|---------| -|`enabled` | Enables custom configuration for modsecurity, defaulting to the [Core Rule Set](https://coreruleset.org/) | `false` | -|`secRuleEngine` | Configures the [rules engine](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#secruleengine) | `DetectionOnly` | -|`secRules` | Creates one or more additional [rule](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRule) | `nil` | - -In the following `auto-deploy-values.yaml` example, some custom settings -are enabled for ModSecurity. Those include setting its engine to -process rules instead of only logging them, while adding two specific -rules which are header-based: - -```yaml -ingress: - modSecurity: - enabled: true - secRuleEngine: "On" - secRules: - - variable: "REQUEST_HEADERS:User-Agent" - operator: "printer" - action: "log,deny,id:'2010',status:403,msg:'printer is an invalid agent'" - - variable: "REQUEST_HEADERS:Content-Type" - operator: "text/plain" - action: "log,deny,id:'2011',status:403,msg:'Text is not supported as content type'" -``` - -#### Running commands in the container - -Applications built with [Auto Build](#auto-build) using Herokuish, the default -unless you have [a custom Dockerfile](#auto-build-using-a-dockerfile), may require -commands to be wrapped as follows: - -```shell -/bin/herokuish procfile exec $COMMAND -``` - -This might be necessary, for example, when: - -- Attaching using `kubectl exec`. -- Using GitLab's [Web Terminal](../../ci/environments.md#web-terminals). - -For example, to start a Rails console from the application root directory, run: - -```shell -/bin/herokuish procfile exec bin/rails c -``` - -### Auto Monitoring - -Once your application is deployed, Auto Monitoring makes it possible to monitor -your application's server and response metrics right out of the box. Auto -Monitoring uses [Prometheus](../../user/project/integrations/prometheus.md) to -get system metrics such as CPU and memory usage directly from -[Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md), -and response metrics such as HTTP error rates, latency, and throughput from the -[NGINX server](../../user/project/integrations/prometheus_library/nginx_ingress.md). - -The metrics include: - -- **Response Metrics:** latency, throughput, error rate -- **System Metrics:** CPU utilization, memory utilization - -To make use of Auto Monitoring: - -1. [Install and configure the requirements](#requirements). -1. [Enable Auto DevOps](#enablingdisabling-auto-devops) if you haven't done already. -1. Finally, go to your project's **CI/CD > Pipelines** and run a pipeline. -1. Once the pipeline finishes successfully, open the - [monitoring dashboard for a deployed environment](../../ci/environments.md#monitoring-environments) - to view the metrics of your deployed application. To view the metrics of the - whole Kubernetes cluster, navigate to **Operations > Metrics**. - - - ## Customizing While Auto DevOps provides great defaults to get you started, you can customize @@ -1099,7 +552,7 @@ If you only require a subset of the features offered by Auto DevOps, you can inc individual Auto DevOps jobs into your own `.gitlab-ci.yml`. Each component job relies on a stage that should be defined in the `.gitlab-ci.yml` that includes the template. -For example, to make use of [Auto Build](#auto-build), you can add the following to +For example, to make use of [Auto Build](stages.md#auto-build), you can add the following to your `.gitlab-ci.yml`: ```yaml @@ -1189,7 +642,7 @@ applications. |-----------------------------------------|------------------------------------| | `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | | `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | -| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](#auto-build-using-cloud-native-buildpacks-beta). | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). | | `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes will not prevent word splitting. [More details](#passing-arguments-to-docker-build). | | `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. | | `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). | diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 9d37f6e12d0..ba6e627cd8b 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -144,33 +144,33 @@ The pipeline is split into 4 stages, each running a couple of jobs.  In the **build** stage, the application is built into a Docker image and then -uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](index.md#auto-build)). +uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](stages.md#auto-build)). In the **test** stage, GitLab runs various checks on the application: - The `test` job runs unit and integration tests by detecting the language and - framework ([Auto Test](index.md#auto-test)) + framework ([Auto Test](stages.md#auto-test)) - The `code_quality` job checks the code quality and is allowed to fail - ([Auto Code Quality](index.md#auto-code-quality-starter)) **(STARTER)** + ([Auto Code Quality](stages.md#auto-code-quality-starter)) **(STARTER)** - The `container_scanning` job checks the Docker container if it has any - vulnerabilities and is allowed to fail ([Auto Container Scanning](index.md#auto-container-scanning-ultimate)) + vulnerabilities and is allowed to fail ([Auto Container Scanning](stages.md#auto-container-scanning-ultimate)) - The `dependency_scanning` job checks if the application has any dependencies - susceptible to vulnerabilities and is allowed to fail ([Auto Dependency Scanning](index.md#auto-dependency-scanning-ultimate)) **(ULTIMATE)** + susceptible to vulnerabilities and is allowed to fail ([Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate)) **(ULTIMATE)** - The `sast` job runs static analysis on the current code to check for potential - security issues and is allowed to fail([Auto SAST](index.md#auto-sast-ultimate)) **(ULTIMATE)** + security issues and is allowed to fail([Auto SAST](stages.md#auto-sast-ultimate)) **(ULTIMATE)** - The `license_management` job searches the application's dependencies to determine each of their - licenses and is allowed to fail ([Auto License Compliance](index.md#auto-license-compliance-ultimate)) **(ULTIMATE)** + licenses and is allowed to fail ([Auto License Compliance](stages.md#auto-license-compliance-ultimate)) **(ULTIMATE)** NOTE: **Note:** As you might have noticed, all jobs except `test` are allowed to fail in the test stage. The **production** stage is run after the tests and checks finish, and it automatically -deploys the application in Kubernetes ([Auto Deploy](index.md#auto-deploy)). +deploys the application in Kubernetes ([Auto Deploy](stages.md#auto-deploy)). Lastly, in the **performance** stage, some performance tests will run on the deployed application -([Auto Browser Performance Testing](index.md#auto-browser-performance-testing-premium)). **(PREMIUM)** +([Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium)). **(PREMIUM)** --- @@ -260,7 +260,7 @@ Let's fix that: 1. Write a commit message and click **Commit**. Now, if you go back to the merge request you should not only see the test passing, but -also the application deployed as a [review app](index.md#auto-review-apps). You +also the application deployed as a [review app](stages.md#auto-review-apps). You can visit it by following clicking the **View app** button. You will see the changes that we previously made. diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md new file mode 100644 index 00000000000..434a30021d5 --- /dev/null +++ b/doc/topics/autodevops/stages.md @@ -0,0 +1,546 @@ +# Stages of Auto DevOps + +The following sections describe the stages of Auto DevOps. Read them carefully +to understand how each one works. + +## Auto Build + +Auto Build creates a build of the application using an existing `Dockerfile` or +Heroku buildpacks. + +Either way, the resulting Docker image is automatically pushed to the +[Container Registry](../../user/packages/container_registry/index.md) and tagged with the commit SHA or tag. + +### Auto Build using a Dockerfile + +If a project's repository contains a `Dockerfile` at its root, Auto Build will use +`docker build` to create a Docker image. + +If you are also using Auto Review Apps and Auto Deploy and choose to provide +your own `Dockerfile`, make sure you expose your application to port +`5000` as this is the port assumed by the +[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](index.md#custom-helm-chart) + +### Auto Build using Heroku buildpacks + +Auto Build builds an application using a project's `Dockerfile` if present, or +otherwise it will use [Herokuish](https://github.com/gliderlabs/herokuish) +and [Heroku buildpacks](https://devcenter.heroku.com/articles/buildpacks) +to automatically detect and build the application into a Docker image. + +Each buildpack requires certain files to be in your project's repository for +Auto Build to successfully build your application. For example, the following +files are required at the root of your application's repository, depending on +the language: + +- A `Pipfile` or `requirements.txt` file for Python projects. +- A `Gemfile` or `Gemfile.lock` file for Ruby projects. + +For the requirements of other languages and frameworks, read the +[buildpacks docs](https://devcenter.heroku.com/articles/buildpacks#officially-supported-buildpacks). + +TIP: **Tip:** +If Auto Build fails despite the project meeting the buildpack requirements, set +a project variable `TRACE=true` to enable verbose logging, which may help to +troubleshoot. + +### Auto Build using Cloud Native Buildpacks (beta) + +> Introduced in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28165). + +Auto Build supports building your application using [Cloud Native Buildpacks](https://buildpacks.io) +through the [`pack` command](https://github.com/buildpacks/pack). To use Cloud Native Buildpacks, +set the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` to a non-empty value. + +Cloud Native Buildpacks (CNBs) are an evolution of Heroku buildpacks, and +will eventually supersede Herokuish-based builds within Auto DevOps. For more +information, see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212692). + +Builds using Cloud Native Buildpacks support the same options as builds using +Heroku buildpacks, with the following caveats: + +- The buildpack must be a Cloud Native Buildpack. A Heroku buildpack can be + converted to a Cloud Native Buildpack using Heroku's + [`cnb-shim`](https://github.com/heroku/cnb-shim). +- `BUILDPACK_URL` must be in a form + [supported by `pack`](https://buildpacks.io/docs/app-developer-guide/specific-buildpacks/). +- The `/bin/herokuish` command is not present in the resulting image, and prefixing + commands with `/bin/herokuish procfile exec` is no longer required (nor possible). + +NOTE: **Note**: Auto Test still uses Herokuish, as test suite detection is not +yet part of the Cloud Native Buildpack specification. For more information, see +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212689). + +## Auto Test + +Auto Test automatically runs the appropriate tests for your application using +[Herokuish](https://github.com/gliderlabs/herokuish) and [Heroku +buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing +your project to detect the language and framework. Several languages and +frameworks are detected automatically, but if your language is not detected, +you may succeed with a [custom buildpack](index.md#custom-buildpacks). Check the +[currently supported languages](index.md#currently-supported-languages). + +Auto Test uses tests you already have in your application. If there are no +tests, it's up to you to add them. + +## Auto Code Quality **(STARTER)** + +Auto Code Quality uses the +[Code Quality image](https://gitlab.com/gitlab-org/ci-cd/codequality) to run +static analysis and other code checks on the current code. The report is +created, and is uploaded as an artifact which you can later download and check +out. + +Any differences between the source and target branches are also +[shown in the merge request widget](../../user/project/merge_requests/code_quality.md). + +## Auto SAST **(ULTIMATE)** + +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. + +Static Application Security Testing (SAST) uses the +[SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static +analysis on the current code and checks for potential security issues. The +Auto SAST stage will be skipped on licenses other than Ultimate and requires GitLab Runner 11.5 or above. + +Once the report is created, it's uploaded as an artifact which you can later download and +check out. + +Any security warnings are also shown in the merge request widget. Read more how +[SAST works](../../user/application_security/sast/index.md). + +## Auto Dependency Scanning **(ULTIMATE)** + +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. + +Dependency Scanning uses the +[Dependency Scanning Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning) +to run analysis on the project dependencies and checks for potential security issues. +The Auto Dependency Scanning stage will be skipped on licenses other than Ultimate +and requires GitLab Runner 11.5 or above. + +Once the +report is created, it's uploaded as an artifact which you can later download and +check out. + +Any security warnings are also shown in the merge request widget. Read more about +[Dependency Scanning](../../user/application_security/dependency_scanning/index.md). + +## Auto License Compliance **(ULTIMATE)** + +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. + +License Compliance uses the +[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) +to search the project dependencies for their license. The Auto License Compliance stage +will be skipped on licenses other than Ultimate. + +Once the +report is created, it's uploaded as an artifact which you can later download and +check out. + +Any licenses are also shown in the merge request widget. Read more how +[License Compliance works](../../user/compliance/license_compliance/index.md). + +## Auto Container Scanning **(ULTIMATE)** + +> Introduced in GitLab 10.4. + +Vulnerability Static Analysis for containers uses +[Clair](https://github.com/quay/clair) to run static analysis on a +Docker image and checks for potential security issues. The Auto Container Scanning stage +will be skipped on licenses other than Ultimate. + +Once the report is +created, it's uploaded as an artifact which you can later download and +check out. + +Any security warnings are also shown in the merge request widget. Read more how +[Container Scanning works](../../user/application_security/container_scanning/index.md). + +## Auto Review Apps + +This is an optional step, since many projects do not have a Kubernetes cluster +available. If the [requirements](index.md#requirements) are not met, the job will +silently be skipped. + +[Review Apps](../../ci/review_apps/index.md) are temporary application environments based on the +branch's code so developers, designers, QA, product managers, and other +reviewers can actually see and interact with code changes as part of the review +process. Auto Review Apps create a Review App for each branch. + +Auto Review Apps will deploy your app to your Kubernetes cluster only. When no cluster +is available, no deployment will occur. + +The Review App will have a unique URL based on the project ID, the branch or tag +name, and a unique number, combined with the Auto DevOps base domain. For +example, `13083-review-project-branch-123456.example.com`. A link to the Review App shows +up in the merge request widget for easy discovery. When the branch or tag is deleted, +for example after the merge request is merged, the Review App will automatically +be deleted. + +Review apps are deployed using the +[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with +Helm, which can be [customized](index.md#custom-helm-chart). The app will be deployed into the [Kubernetes +namespace](../../user/project/clusters/index.md#deployment-variables) +for the environment. + +Since GitLab 11.4, a [local +Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is +used. Previous versions of GitLab had a Tiller installed in the project +namespace. + +CAUTION: **Caution:** +Your apps should *not* be manipulated outside of Helm (using Kubernetes directly). +This can cause confusion with Helm not detecting the change and subsequent +deploys with Auto DevOps can undo your changes. Also, if you change something +and want to undo it by deploying again, Helm may not detect that anything changed +in the first place, and thus not realize that it needs to re-apply the old config. + +## Auto DAST **(ULTIMATE)** + +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. + +Dynamic Application Security Testing (DAST) uses the +popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) +to perform an analysis on the current code and checks for potential security +issues. The Auto DAST stage will be skipped on licenses other than Ultimate. + +Once the DAST scan is complete, any security warnings are shown +on the [Security Dashboard](../../user/application_security/security_dashboard/index.md) +and the Merge Request Widget. Read how +[DAST works](../../user/application_security/dast/index.md). + +On your default branch, DAST scans an app deployed specifically for that purpose. +The app is deleted after DAST has run. + +On feature branches, DAST scans the [review app](#auto-review-apps). + +### Overriding the DAST target + +To use a custom target instead of the auto-deployed review apps, +set a `DAST_WEBSITE` environment variable to the URL for DAST to scan. + +NOTE: **Note:** +If [DAST Full Scan](../../user/application_security/dast/index.md#full-scan) is enabled, it is strongly advised **not** +to set `DAST_WEBSITE` to any staging or production environment. DAST Full Scan +actively attacks the target, which can take down the application and lead to +data loss or corruption. + +### Disabling Auto DAST + +DAST can be disabled: + +- On all branches by setting the `DAST_DISABLED` environment variable to `"true"`. +- Only on the default branch by setting the `DAST_DISABLED_FOR_DEFAULT_BRANCH` environment variable to `"true"`. + +## Auto Browser Performance Testing **(PREMIUM)** + +> Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. + +Auto Browser Performance Testing utilizes the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) to measure the performance of a web page. A JSON report is created and uploaded as an artifact, which includes the overall performance score for each page. By default, the root page of Review and Production environments will be tested. If you would like to add additional URL's to test, simply add the paths to a file named `.gitlab-urls.txt` in the root directory, one per line. For example: + +```text +/ +/features +/direction +``` + +Any performance differences between the source and target branches are also +[shown in the merge request widget](../../user/project/merge_requests/browser_performance_testing.md). + +## Auto Deploy + +This is an optional step, since many projects do not have a Kubernetes cluster +available. If the [requirements](index.md#requirements) are not met, the job will +silently be skipped. + +After a branch or merge request is merged into the project's default branch (usually +`master`), Auto Deploy deploys the application to a `production` environment in +the Kubernetes cluster, with a namespace based on the project name and unique +project ID, for example `project-4321`. + +Auto Deploy doesn't include deployments to staging or canary by default, but the +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to +enable them. + +You can make use of [environment variables](index.md#environment-variables) to automatically +scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](index.md#custom-helm-chart). + +Apps are deployed using the +[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with +Helm. The app will be deployed into the [Kubernetes +namespace](../../user/project/clusters/index.md#deployment-variables) +for the environment. + +Since GitLab 11.4, a [local +Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is +used. Previous versions of GitLab had a Tiller installed in the project +namespace. + +CAUTION: **Caution:** +Your apps should *not* be manipulated outside of Helm (using Kubernetes directly). +This can cause confusion with Helm not detecting the change and subsequent +deploys with Auto DevOps can undo your changes. Also, if you change something +and want to undo it by deploying again, Helm may not detect that anything changed +in the first place, and thus not realize that it needs to re-apply the old config. + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19507) in GitLab 11.0. + +For internal and private projects a [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token) +will be automatically created, when Auto DevOps is enabled and the Auto DevOps settings are saved. This Deploy Token +can be used for permanent access to the registry. When the GitLab Deploy Token has been manually revoked, it won't be automatically created. + +If the GitLab Deploy Token cannot be found, `CI_REGISTRY_PASSWORD` is +used. Note that `CI_REGISTRY_PASSWORD` is only valid during deployment. +This means that Kubernetes will be able to successfully pull the +container image during deployment but in cases where the image needs to +be pulled again, e.g. after pod eviction, Kubernetes will fail to do so +as it will be attempting to fetch the image using +`CI_REGISTRY_PASSWORD`. + +### Kubernetes 1.16+ + +> - [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/51) in GitLab 12.8. +> - Support for deploying a PostgreSQL version that supports Kubernetes 1.16+ was [introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/49) in GitLab 12.9. + +CAUTION: **Deprecation** +The default value of `extensions/v1beta1` for the `deploymentApiVersion` setting is +deprecated, and is scheduled to be changed to a new default of `apps/v1` in +[GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/issues/47). + +In Kubernetes 1.16 onwards, a number of [APIs were removed](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/), +including support for `Deployment` in the `extensions/v1beta1` version. + +To use Auto Deploy on a Kubernetes 1.16+ cluster, you must: + +1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart): + + ```yml + deploymentApiVersion: apps/v1 + ``` + +1. Set the: + + - `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`. + - `POSTGRES_VERSION` variable to `9.6.16` or higher. + + This will opt-in to using a version of the PostgreSQL chart that supports Kubernetes + 1.16 and higher. + +DANGER: **Danger:** Opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version +`2` will delete the version `1` PostgreSQL database. Please follow the +guide on [upgrading PostgreSQL](upgrading_postgresql.md) to backup and +restore your database before opting into version `2`. + +### Migrations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21955) in GitLab 11.4 + +Database initialization and migrations for PostgreSQL can be configured to run +within the application pod by setting the project variables `DB_INITIALIZE` and +`DB_MIGRATE` respectively. + +If present, `DB_INITIALIZE` will be run as a shell command within an +application pod as a Helm post-install hook. As some applications will +not run without a successful database initialization step, GitLab will +deploy the first release without the application deployment and only the +database initialization step. After the database initialization completes, +GitLab will deploy a second release with the application deployment as +normal. + +Note that a post-install hook means that if any deploy succeeds, +`DB_INITIALIZE` will not be processed thereafter. + +If present, `DB_MIGRATE` will be run as a shell command within an application pod as +a Helm pre-upgrade hook. + +For example, in a Rails application in an image built with +[Herokuish](https://github.com/gliderlabs/herokuish): + +- `DB_INITIALIZE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:setup` +- `DB_MIGRATE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:migrate` + +Unless you have a `Dockerfile` in your repo, your image is built with +Herokuish, and you must prefix commands run in these images with `/bin/herokuish +procfile exec` to replicate the environment where your application will run. + +### Workers + +Some web applications need to run extra deployments for "worker processes". For +example, it is common in a Rails application to have a separate worker process +to run background tasks like sending emails. + +The [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) +used in Auto Deploy [has support for running worker +processes](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/9). + +In order to run a worker, you'll need to ensure that it is able to respond to +the standard health checks, which expect a successful HTTP response on port +`5000`. For [Sidekiq](https://github.com/mperham/sidekiq), you could make use of +the [`sidekiq_alive` gem](https://rubygems.org/gems/sidekiq_alive) to do this. + +In order to work with Sidekiq, you'll also need to ensure your deployments have +access to a Redis instance. Auto DevOps won't deploy this for you so you'll +need to: + +- Maintain your own Redis instance. +- Set a CI variable `K8S_SECRET_REDIS_URL`, which the URL of this instance to + ensure it's passed into your deployments. + +Once you have configured your worker to respond to health checks, run a Sidekiq +worker for your Rails application. You can enable workers by setting the +following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart): + +```yml +workers: + sidekiq: + replicaCount: 1 + command: + - /bin/herokuish + - procfile + - exec + - sidekiq + preStopCommand: + - /bin/herokuish + - procfile + - exec + - sidekiqctl + - quiet + terminationGracePeriodSeconds: 60 +``` + +### Network Policy + +> [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/30) in GitLab 12.7. + +By default, all Kubernetes pods are +[non-isolated](https://kubernetes.io/docs/concepts/services-networking/network-policies/#isolated-and-non-isolated-pods), +meaning that they will accept traffic to and from any source. You can use +[NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +to restrict connections to and from selected pods, namespaces, and the Internet. + +NOTE: **Note:** +You must use a Kubernetes network plugin that implements support for +`NetworkPolicy`. The default network plugin for Kubernetes (`kubenet`) +[does not implement](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#kubenet) +support for it. The [Cilium](https://cilium.io/) network plugin can be +installed as a [cluster application](../../user/clusters/applications.md#install-cilium-using-gitlab-cicd) +to enable support for network policies. + +You can enable deployment of a network policy by setting the following +in the `.gitlab/auto-deploy-values.yaml` file: + +```yaml +networkPolicy: + enabled: true +``` + +The default policy deployed by the auto deploy pipeline will allow +traffic within a local namespace and from the `gitlab-managed-apps` +namespace. All other inbound connection will be blocked. Outbound +traffic (for example, to the Internet) is not affected by the default policy. + +You can also provide a custom [policy specification](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#networkpolicyspec-v1-networking-k8s-io) +via the `.gitlab/auto-deploy-values.yaml` file, for example: + +```yaml +networkPolicy: + enabled: true + spec: + podSelector: + matchLabels: + app.gitlab.com/env: staging + ingress: + - from: + - podSelector: + matchLabels: {} + - namespaceSelector: + matchLabels: + app.gitlab.com/managed_by: gitlab +``` + +For more information on how to install Network Policies, see +[Install Cilium using GitLab CI/CD](../../user/clusters/applications.md#install-cilium-using-gitlab-cicd). + +### Web Application Firewall (ModSecurity) customization + +> [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/44) in GitLab 12.8. + +Customization on an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) or on a deployment base is available for clusters with [ModSecurity installed](../../user/clusters/applications.md#web-application-firewall-modsecurity). + +To enable ModSecurity with Auto Deploy, you need to create a `.gitlab/auto-deploy-values.yaml` file in your project with the following attributes. + +|Attribute | Description | Default | +-----------|-------------|---------| +|`enabled` | Enables custom configuration for modsecurity, defaulting to the [Core Rule Set](https://coreruleset.org/) | `false` | +|`secRuleEngine` | Configures the [rules engine](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#secruleengine) | `DetectionOnly` | +|`secRules` | Creates one or more additional [rule](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRule) | `nil` | + +In the following `auto-deploy-values.yaml` example, some custom settings +are enabled for ModSecurity. Those include setting its engine to +process rules instead of only logging them, while adding two specific +rules which are header-based: + +```yaml +ingress: + modSecurity: + enabled: true + secRuleEngine: "On" + secRules: + - variable: "REQUEST_HEADERS:User-Agent" + operator: "printer" + action: "log,deny,id:'2010',status:403,msg:'printer is an invalid agent'" + - variable: "REQUEST_HEADERS:Content-Type" + operator: "text/plain" + action: "log,deny,id:'2011',status:403,msg:'Text is not supported as content type'" +``` + +### Running commands in the container + +Applications built with [Auto Build](#auto-build) using Herokuish, the default +unless you have [a custom Dockerfile](#auto-build-using-a-dockerfile), may require +commands to be wrapped as follows: + +```shell +/bin/herokuish procfile exec $COMMAND +``` + +This might be necessary, for example, when: + +- Attaching using `kubectl exec`. +- Using GitLab's [Web Terminal](../../ci/environments.md#web-terminals). + +For example, to start a Rails console from the application root directory, run: + +```shell +/bin/herokuish procfile exec bin/rails c +``` + +## Auto Monitoring + +Once your application is deployed, Auto Monitoring makes it possible to monitor +your application's server and response metrics right out of the box. Auto +Monitoring uses [Prometheus](../../user/project/integrations/prometheus.md) to +get system metrics such as CPU and memory usage directly from +[Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md), +and response metrics such as HTTP error rates, latency, and throughput from the +[NGINX server](../../user/project/integrations/prometheus_library/nginx_ingress.md). + +The metrics include: + +- **Response Metrics:** latency, throughput, error rate +- **System Metrics:** CPU utilization, memory utilization + +To make use of Auto Monitoring: + +1. [Install and configure the requirements](index.md#requirements). +1. [Enable Auto DevOps](index.md#enablingdisabling-auto-devops) if you haven't done already. +1. Finally, go to your project's **CI/CD > Pipelines** and run a pipeline. +1. Once the pipeline finishes successfully, open the + [monitoring dashboard for a deployed environment](../../ci/environments.md#monitoring-environments) + to view the metrics of your deployed application. To view the metrics of the + whole Kubernetes cluster, navigate to **Operations > Metrics**. + + diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md index 13871eea4af..768293a6153 100644 --- a/doc/topics/web_application_firewall/quick_start_guide.md +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -150,12 +150,12 @@ The pipeline is split into a few stages, each running a couple of jobs.  In the **build** stage, the application is built into a Docker image and then -uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](../autodevops/index.md#auto-build)). +uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](../autodevops/stages.md#auto-build)). In the **test** stage, GitLab runs various checks on the application. The **production** stage is run after the tests and checks finish, and it automatically -deploys the application in Kubernetes ([Auto Deploy](../autodevops/index.md#auto-deploy)). +deploys the application in Kubernetes ([Auto Deploy](../autodevops/stages.md#auto-deploy)). The **production** stage creates Kubernetes objects like a Deployment, Service, and Ingress resource. The diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 610e11b18a9..e2af69c19e9 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -15,7 +15,7 @@ two open source tools for Vulnerability Static Analysis for containers. You can take advantage of Container Scanning by either [including the CI job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto Container Scanning](../../../topics/autodevops/index.md#auto-container-scanning-ultimate) +[Auto Container Scanning](../../../topics/autodevops/stages.md#auto-container-scanning-ultimate) that is provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the Container Scanning report, compares the found vulnerabilities diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index e60cf095f3f..f2b7c30b5cc 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -24,7 +24,7 @@ for known vulnerabilities using Dynamic Application Security Testing (DAST). You can take advantage of DAST by either [including the CI job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto DAST](../../../topics/autodevops/index.md#auto-dast-ultimate) +[Auto DAST](../../../topics/autodevops/stages.md#auto-dast-ultimate) that is provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the DAST report, compares the found vulnerabilities between the source and target diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 651a7730cdb..faf885b6552 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -18,7 +18,7 @@ All dependencies are scanned, including the transitive dependencies (also known You can take advantage of Dependency Scanning by either [including the CI job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto Dependency Scanning](../../../topics/autodevops/index.md#auto-dependency-scanning-ultimate) +[Auto Dependency Scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning-ultimate) that is provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the Dependency Scanning report, compares the found vulnerabilities diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 9c6098e4e04..c864abfe1ae 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -19,7 +19,7 @@ vulnerabilities using Static Application Security Testing (SAST). You can take advantage of SAST by doing one of the following: - [Including the CI job](#configuration) in your existing `.gitlab-ci.yml` file. -- Implicitly using [Auto SAST](../../../topics/autodevops/index.md#auto-sast-ultimate) provided by +- Implicitly using [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the SAST report, compares the found vulnerabilities between the @@ -96,7 +96,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Configuration NOTE: **Note:** -You don't have to configure SAST manually as shown in this section if you're using [Auto SAST](../../../topics/autodevops/index.md#auto-sast-ultimate) +You don't have to configure SAST manually as shown in this section if you're using [Auto SAST](../../../topics/autodevops/stages.md#auto-sast-ultimate) provided by [Auto DevOps](../../../topics/autodevops/index.md). For GitLab 11.9 and later, to enable SAST you must [include](../../../ci/yaml/README.md#includetemplate) diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index 3a3e2b36133..482fceea680 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -13,7 +13,7 @@ navigating to your project's **Security & Compliance > Threat Monitoring** page. GitLab supports statistics for the following security features: - [Web Application Firewall](../../clusters/applications.md#web-application-firewall-modsecurity) -- [Container Network Policies](../../../topics/autodevops/index.md#network-policy) +- [Container Network Policies](../../../topics/autodevops/stages.md#network-policy) ## Web Application Firewall diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 08ac82ed781..0d2a3ac3ed9 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -754,7 +754,7 @@ available configuration options. [Cilium](https://cilium.io/) is a networking plugin for Kubernetes that you can use to implement support for [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -resources. For more information on [Network Policies](../../topics/autodevops/index.md#network-policy), see the documentation. +resources. For more information on [Network Policies](../../topics/autodevops/stages.md#network-policy), see the documentation. Enable Cilium in the `.gitlab/managed-apps/config.yaml` file to install it: diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 87dc5406c89..4a90cc0a72f 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -13,7 +13,7 @@ using License Compliance. You can take advantage of License Compliance by either [including the job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto License Compliance](../../../topics/autodevops/index.md#auto-license-compliance-ultimate) +[Auto License Compliance](../../../topics/autodevops/stages.md#auto-license-compliance-ultimate) that is provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the License Compliance report, compares the licenses between the diff --git a/doc/user/group/saml_sso/img/scim_advanced.png b/doc/user/group/saml_sso/img/scim_advanced.png Binary files differdeleted file mode 100644 index c9e095dc89a..00000000000 --- a/doc/user/group/saml_sso/img/scim_advanced.png +++ /dev/null diff --git a/doc/user/group/saml_sso/img/scim_attribute_mapping.png b/doc/user/group/saml_sso/img/scim_attribute_mapping.png Binary files differdeleted file mode 100644 index 933d8fb6f36..00000000000 --- a/doc/user/group/saml_sso/img/scim_attribute_mapping.png +++ /dev/null diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 1243cf7c2f5..7621679c6cc 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -415,7 +415,9 @@ Alternatively, an admin of your Identity Provider can use the [SCIM API](../../. ### Message: "SAML authentication failed: Email has already been taken" -Same as ["SAML authentication failed: User has already been taken"](#message-saml-authentication-failed-user-has-already-been-taken). +| Cause | Solution | +|------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------| +| When a user account with the email address already exists in GitLab, but the user does not have the SAML identity tied to their account. | The user will need to [link their account](#user-access-and-management). | ### Message: "SAML authentication failed: Extern uid has already been taken, User has already been taken" diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index e38cc73d035..e4dff12b004 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -62,7 +62,7 @@ You can then test the connection by clicking on **Test Connection**. If the conn #### Configure attribute mapping -1. Click on `Synchronize Azure Active Directory Users to AppName`, to configure the attribute mapping. +1. Click on `Synchronize Azure Active Directory Users to AppName` to configure the attribute mapping. 1. Click **Delete** next to the `mail` mapping. 1. Map `userPrincipalName` to `emails[type eq "work"].value` and change its **Matching precedence** to `2`. 1. Map `mailNickname` to `userName`. @@ -74,33 +74,25 @@ You can then test the connection by clicking on **Test Connection**. If the conn 1. Create a new mapping: 1. Click **Add New Mapping**. 1. Set: - - **Source attribute** to the unique identifier determined above. - - **Target attribute** to `id`. + - **Source attribute** to the unique identifier determined above, typically `objectId`. + - **Target attribute** to `externalId`. - **Match objects using this attribute** to `Yes`. - **Matching precedence** to `1`. -1. Create another new mapping: - 1. Click **Add New Mapping**. - 1. Set: - - **Source attribute** to the unique identifier determined above. - - **Target attribute** to `externalId`. -1. Click the `userPrincipalName` mapping and change **Match objects using this attribute** to `No`. - Save your changes and you should have the following configuration: +1. Click the `userPrincipalName` mapping and change **Match objects using this attribute** to `No`. -  +1. Save your changes. For reference, you can view [an example configuration in the troubleshooting reference](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory). - NOTE: **Note:** If you used a unique identifier **other than** `objectId`, be sure to map it instead to both `id` and `externalId`. + NOTE: **Note:** If you used a unique identifier **other than** `objectId`, be sure to map it to `externalId`. 1. Below the mapping list click on **Show advanced options > Edit attribute list for AppName**. -1. Leave the `id` as the primary and only required field. +1. Ensure the `id` is the primary and required field, and `externalId` is also required. NOTE: **Note:** `username` should neither be primary nor required as we don't support that field on GitLab SCIM yet. -  - 1. Save all the screens and, in the **Provisioning** step, set the `Provisioning Status` to `On`. diff --git a/doc/user/index.md b/doc/user/index.md index b6defdc3f65..ac19c5a0de7 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -88,7 +88,7 @@ it all at once, from one single project. Use built-in [GitLab CI/CD](../ci/README.md) to test, build, and deploy your applications directly from GitLab. No third-party integrations needed. -- [GitLab Auto Deploy](../topics/autodevops/index.md#auto-deploy): Deploy your application out-of-the-box with GitLab Auto Deploy. +- [GitLab Auto Deploy](../topics/autodevops/stages.md#auto-deploy): Deploy your application out-of-the-box with GitLab Auto Deploy. - [Review Apps](../ci/review_apps/index.md): Live-preview the changes introduced by a merge request with Review Apps. - [GitLab Pages](project/pages/index.md): Publish your static site directly from GitLab with GitLab Pages. You can build, test, and deploy any Static Site Generator with Pages. diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index f12a281584f..86e9df1d162 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -44,7 +44,7 @@ Canary deployments require that you properly configure Deploy Boards: 1. Follow the steps to [enable Deploy Boards](deploy_boards.md#enabling-deploy-boards). 1. To track canary deployments you need to label your Kubernetes deployments and - pods with `track: canary`. To get started quickly, you can use the [Auto Deploy](../../topics/autodevops/index.md#auto-deploy) + pods with `track: canary`. To get started quickly, you can use the [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy) template for canary deployments that GitLab provides. Depending on the deploy, the label should be either `stable` or `canary`. diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 019ba083584..c479f610ff1 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -81,7 +81,7 @@ To display the Deploy Boards for a specific [environment](../../ci/environments. `$CI_PROJECT_PATH_SLUG` are the values of the CI variables. This is so we can lookup the proper environment in a cluster/namespace which may have more than one. These resources should be contained in the namespace defined in - the Kubernetes service setting. You can use an [Autodeploy](../../topics/autodevops/index.md#auto-deploy) `.gitlab-ci.yml` + the Kubernetes service setting. You can use an [Autodeploy](../../topics/autodevops/stages.md#auto-deploy) `.gitlab-ci.yml` template which has predefined stages and commands to use, and automatically applies the annotations. Each project will need to have a unique namespace in Kubernetes as well. The image below demonstrates how this is shown inside @@ -139,7 +139,7 @@ version of your application. ## Further reading -- [GitLab Autodeploy](../../topics/autodevops/index.md#auto-deploy) +- [GitLab Autodeploy](../../topics/autodevops/stages.md#auto-deploy) - [GitLab CI/CD environment variables](../../ci/variables/README.md) - [Environments and deployments](../../ci/environments.md) - [Kubernetes deploy example](https://gitlab.com/gitlab-examples/kubernetes-deploy) diff --git a/doc/user/project/index.md b/doc/user/project/index.md index f5bb2ee78d9..3e1c3007e00 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -63,7 +63,7 @@ When you create a project in GitLab, you'll have access to a large number of - [GitLab CI/CD](../../ci/README.md): GitLab's built-in [Continuous Integration, Delivery, and Deployment](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/) tool - [Container Registry](../packages/container_registry/index.md): Build and push Docker images out-of-the-box - - [Auto Deploy](../../topics/autodevops/index.md#auto-deploy): Configure GitLab CI/CD + - [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy): Configure GitLab CI/CD to automatically set up your app's deployment - [Enable and disable GitLab CI/CD](../../ci/enable_or_disable_ci.md) - [Pipelines](../../ci/pipelines/index.md): Configure and visualize diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 7a21887d29f..eb0013ab6e5 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -76,7 +76,7 @@ The Prometheus server will [automatically detect and monitor](https://prometheus - `prometheus.io/port` to define the port of the metrics endpoint. - `prometheus.io/path` to define the path of the metrics endpoint. Defaults to `/metrics`. -CPU and Memory consumption is monitored, but requires [naming conventions](prometheus_library/kubernetes.md#specifying-the-environment) in order to determine the environment. If you are using [Auto DevOps](../../../topics/autodevops/), this is handled automatically. +CPU and Memory consumption is monitored, but requires [naming conventions](prometheus_library/kubernetes.md#specifying-the-environment) in order to determine the environment. If you are using [Auto DevOps](../../../topics/autodevops/index.md), this is handled automatically. The [NGINX Ingress](../clusters/index.md#installing-applications) that is deployed by GitLab to clusters, is automatically annotated for monitoring providing key response metrics: latency, throughput, and error rates. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index 4d54dcd4c4c..62f8c08e298 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -55,4 +55,4 @@ Managing these settings depends on how NGINX Ingress has been deployed. If you h In order to isolate and only display relevant metrics for a given environment, GitLab needs a method to detect which labels are associated. To do this, GitLab will search for metrics with appropriate labels. In this case, the `ingress` label must `<CI_ENVIRONMENT_SLUG>`. -If you have used [Auto Deploy](../../../../topics/autodevops/index.md#auto-deploy) to deploy your app, this format will be used automatically and metrics will be detected with no action on your part. +If you have used [Auto Deploy](../../../../topics/autodevops/stages.md#auto-deploy) to deploy your app, this format will be used automatically and metrics will be detected with no action on your part. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md index b00a857f718..af3b725deb6 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md @@ -55,4 +55,4 @@ Managing these settings depends on how NGINX Ingress has been deployed. If you h In order to isolate and only display relevant metrics for a given environment, GitLab needs a method to detect which labels are associated. To do this, GitLab will search for metrics with appropriate labels. In this case, the `upstream` label must be of the form `<KUBE_NAMESPACE>-<CI_ENVIRONMENT_SLUG>-*`. -If you have used [Auto Deploy](../../../../topics/autodevops/index.md#auto-deploy) to deploy your app, this format will be used automatically and metrics will be detected with no action on your part. +If you have used [Auto Deploy](../../../../topics/autodevops/stages.md#auto-deploy) to deploy your app, this format will be used automatically and metrics will be detected with no action on your part. diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index 76c5192a482..ce1e1a31bbe 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -19,7 +19,7 @@ Code Quality: Quality](https://gitlab.com/gitlab-org/ci-cd/codequality) project using [default Code Climate configurations](https://gitlab.com/gitlab-org/ci-cd/codequality/-/tree/master/codeclimate_defaults). - Can make use of a [template](#example-configuration). - Is available with [Auto - DevOps](../../../topics/autodevops/index.md#auto-code-quality-starter). + DevOps](../../../topics/autodevops/stages.md#auto-code-quality-starter). - Can be extended through [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) or a [custom tool](#implementing-a-custom-tool). Going a step further, GitLab can show the Code Quality report right diff --git a/doc/user/project/repository/web_editor.md b/doc/user/project/repository/web_editor.md index 28941e755c0..59fedacb2ca 100644 --- a/doc/user/project/repository/web_editor.md +++ b/doc/user/project/repository/web_editor.md @@ -126,7 +126,7 @@ a blank `README.md` file to it, and creates and redirects you to a new branch based on the issue title. If your [project is already configured with a deployment service](../integrations/overview.md), such as Kubernetes, GitLab takes one step further and prompts you to set up -[auto deploy](../../../topics/autodevops/index.md#auto-deploy) +[auto deploy](../../../topics/autodevops/stages.md#auto-deploy) by helping you create a `.gitlab-ci.yml` file. After the branch is created, you can edit files in the repository to fix diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c400e1cd4fd..f1ca8900c30 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -45,6 +45,8 @@ module Gitlab attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :repository_path, :redirected_path, :auth_result_type, :changes, :logger + alias_method :container, :project + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @project = project @@ -429,7 +431,72 @@ module Gitlab end def repository - project.repository + container&.repository + end + + def check_size_before_push! + if check_size_limit? && size_checker.above_size_limit? + raise ForbiddenError, size_checker.error_message.push_error + end + end + + def check_push_size! + return unless check_size_limit? + + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + repository.clean_stale_repository_files + + # Use #check_repository_disk_size to get correct push size whenever a lot of changes + # gets pushed at the same time containing the same blobs. This is only + # doable if GIT_OBJECT_DIRECTORY_RELATIVE env var is set and happens + # when git push comes from CLI (not via UI and API). + # + # Fallback to determining push size using the changes_list so we can still + # determine the push size if env var isn't set (e.g. changes are made + # via UI and API). + if check_quarantine_size? + check_repository_disk_size + else + check_changes_size + end + end + + def check_quarantine_size? + git_env = ::Gitlab::Git::HookEnv.all(repository.gl_repository) + + git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + end + + def check_repository_disk_size + check_size_against_limit(repository.object_directory_size) + end + + def check_changes_size + changes_size = 0 + + changes_list.each do |change| + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) # rubocop: disable CodeReuse/ActiveRecord + + check_size_against_limit(changes_size) + end + end + + def check_size_against_limit(size) + if size_checker.changes_will_exceed_size_limit?(size) + raise ForbiddenError, size_checker.error_message.new_changes_error + end + end + + def check_size_limit? + strong_memoize(:check_size_limit) do + changes_list.any? { |change| !Gitlab::Git.blank_ref?(change[:newrev]) } + end + end + + def size_checker + container.repository_size_checker end end end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index e11c1ea527c..d05ffe9b508 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -14,6 +14,8 @@ module Gitlab attr_reader :snippet + alias_method :container, :snippet + def initialize(actor, snippet, protocol, **kwargs) @snippet = snippet @@ -53,11 +55,6 @@ module Gitlab check_change_access! end - override :repository - def repository - snippet&.repository - end - def check_snippet_accessibility! if snippet.blank? raise NotFoundError, ERROR_MESSAGES[:snippet_not_found] @@ -89,11 +86,15 @@ module Gitlab raise ForbiddenError, ERROR_MESSAGES[:update_snippet] end + check_size_before_push! + changes_list.each do |change| # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up check_single_change_access(change) end + + check_push_size! end def check_single_change_access(change) diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index cf1af844439..82c57f7a07d 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -5,12 +5,14 @@ module Gitlab class RepositorySizeChecker attr_reader :limit + # @param current_size_proc [Proc] returns repository size in bytes def initialize(current_size_proc:, limit:, enabled: true) @current_size_proc = current_size_proc @limit = limit @enabled = enabled && limit != 0 end + # @return [Integer] bytes def current_size @current_size ||= @current_size_proc.call end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index ad1d79c3733..5fd9e119bc0 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -19025,9 +19025,6 @@ msgstr "" msgid "Specific Runners" msgstr "" -msgid "Specified URL cannot be used." -msgstr "" - msgid "Specified URL cannot be used: \"%{reason}\"" msgstr "" diff --git a/spec/controllers/import/gitea_controller_spec.rb b/spec/controllers/import/gitea_controller_spec.rb index b4834dffdb3..006b423ce5f 100644 --- a/spec/controllers/import/gitea_controller_spec.rb +++ b/spec/controllers/import/gitea_controller_spec.rb @@ -42,7 +42,7 @@ describe Import::GiteaController do get :status, format: :json expect(controller).to redirect_to(new_import_url) - expect(flash[:alert]).to eq('Specified URL cannot be used.') + expect(flash[:alert]).to eq('Specified URL cannot be used: "Only allowed schemes are http, https"') end end end diff --git a/spec/controllers/projects/import/jira_controller_spec.rb b/spec/controllers/projects/import/jira_controller_spec.rb index 33b51320391..f692f976bc0 100644 --- a/spec/controllers/projects/import/jira_controller_spec.rb +++ b/spec/controllers/projects/import/jira_controller_spec.rb @@ -93,18 +93,29 @@ describe Projects::Import::JiraController do end context 'post import' do - it 'creates import state' do - expect(project.import_state).to be_nil + context 'when jira project key is empty' do + it 'redirects back to show with an error' do + post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: '' } - post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: 'Test' } + expect(response).to redirect_to(project_import_jira_path(project)) + expect(flash[:alert]).to eq('No jira project key has been provided.') + end + end - project.reload + context 'when everything is ok' do + it 'creates import state' do + expect(project.import_state).to be_nil - jira_project = project.import_data.data.dig('jira', 'projects').first - expect(project.import_type).to eq 'jira' - expect(project.import_state.status).to eq 'scheduled' - expect(jira_project['key']).to eq 'Test' - expect(response).to redirect_to(project_import_jira_path(project)) + post :import, params: { namespace_id: project.namespace, project_id: project, jira_project_key: 'Test' } + + project.reload + + jira_project = project.import_data.data.dig('jira', 'projects').first + expect(project.import_type).to eq 'jira' + expect(project.import_state.status).to eq 'scheduled' + expect(jira_project['key']).to eq 'Test' + expect(response).to redirect_to(project_import_jira_path(project)) + end end end end diff --git a/spec/lib/gitlab/git_access_snippet_spec.rb b/spec/lib/gitlab/git_access_snippet_spec.rb index f52fe8ef612..bbc3808df12 100644 --- a/spec/lib/gitlab/git_access_snippet_spec.rb +++ b/spec/lib/gitlab/git_access_snippet_spec.rb @@ -11,6 +11,7 @@ describe Gitlab::GitAccessSnippet do let_it_be(:user) { create(:user) } let_it_be(:project) { create(:project, :public) } let_it_be(:snippet) { create(:project_snippet, :public, :repository, project: project) } + let(:repository) { snippet.repository } let(:actor) { user } let(:protocol) { 'ssh' } @@ -211,6 +212,84 @@ describe Gitlab::GitAccessSnippet do end end + describe 'repository size restrictions' do + let(:snippet) { create(:personal_snippet, :public, :repository) } + let(:actor) { snippet.author } + + let(:oldrev) { TestEnv::BRANCH_SHA["snippet/single-file"] } + let(:newrev) { TestEnv::BRANCH_SHA["snippet/edit-file"] } + let(:ref) { "refs/heads/snippet/edit-file" } + let(:changes) { "#{oldrev} #{newrev} #{ref}" } + + shared_examples_for 'a push to repository already over the limit' do + it 'errs' do + expect(snippet.repository_size_checker).to receive(:above_size_limit?).and_return(true) + + expect do + push_access_check + end.to raise_error(described_class::ForbiddenError, /Your push has been rejected/) + end + end + + shared_examples_for 'a push to repository below the limit' do + it 'does not err' do + expect(snippet.repository_size_checker).to receive(:above_size_limit?).and_return(false) + expect(snippet.repository_size_checker) + .to receive(:changes_will_exceed_size_limit?) + .with(change_size) + .and_return(false) + + expect { push_access_check }.not_to raise_error + end + end + + shared_examples_for 'a push to repository to make it over the limit' do + it 'errs' do + expect(snippet.repository_size_checker).to receive(:above_size_limit?).and_return(false) + expect(snippet.repository_size_checker) + .to receive(:changes_will_exceed_size_limit?) + .with(change_size) + .and_return(true) + + expect do + push_access_check + end.to raise_error(described_class::ForbiddenError, /Your push to this repository would cause it to exceed the size limit/) + end + end + + context 'when GIT_OBJECT_DIRECTORY_RELATIVE env var is set' do + let(:change_size) { 100 } + + before do + allow(Gitlab::Git::HookEnv) + .to receive(:all) + .with(repository.gl_repository) + .and_return({ 'GIT_OBJECT_DIRECTORY_RELATIVE' => 'objects' }) + + # Stub the object directory size to "simulate" quarantine size + allow(repository).to receive(:object_directory_size).and_return(change_size) + end + + it_behaves_like 'a push to repository already over the limit' + it_behaves_like 'a push to repository below the limit' + it_behaves_like 'a push to repository to make it over the limit' + end + + context 'when GIT_OBJECT_DIRECTORY_RELATIVE env var is not set' do + let(:change_size) { 200 } + + before do + allow(snippet.repository).to receive(:new_blobs).and_return( + [double(:blob, size: change_size)] + ) + end + + it_behaves_like 'a push to repository already over the limit' + it_behaves_like 'a push to repository below the limit' + it_behaves_like 'a push to repository to make it over the limit' + end + end + private def raise_snippet_not_found diff --git a/spec/models/snippet_spec.rb b/spec/models/snippet_spec.rb index 8ce3718e3c1..9cced1f24ab 100644 --- a/spec/models/snippet_spec.rb +++ b/spec/models/snippet_spec.rb @@ -696,6 +696,23 @@ describe Snippet do end end + describe '#repository_size_checker' do + subject { build(:personal_snippet) } + + let(:checker) { subject.repository_size_checker } + let(:current_size) { 60 } + + before do + allow(subject.repository).to receive(:_uncached_size).and_return(current_size) + end + + it 'sets up size checker', :aggregate_failures do + expect(checker.current_size).to eq(current_size.megabytes) + expect(checker.limit).to eq(Gitlab::CurrentSettings.snippet_size_limit) + expect(checker.enabled?).to be_truthy + end + end + describe '#can_cache_field?' do using RSpec::Parameterized::TableSyntax diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 2e69c59c80a..fd41c5c8fe3 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -60,11 +60,11 @@ module TestEnv 'merge-commit-analyze-before' => '1adbdef', 'merge-commit-analyze-side-branch' => '8a99451', 'merge-commit-analyze-after' => '646ece5', - 'snippet/single-file' => '43e4080', - 'snippet/multiple-files' => 'b80faa8', - 'snippet/rename-and-edit-file' => '220a1e4', - 'snippet/edit-file' => 'c2f074f', - 'snippet/no-files' => '671aaa8', + 'snippet/single-file' => '43e4080aaa14fc7d4b77ee1f5c9d067d5a7df10e', + 'snippet/multiple-files' => 'b80faa8c5b2b62f6489a0d84755580e927e1189b', + 'snippet/rename-and-edit-file' => '220a1e4b4dff37feea0625a7947a4c60fbe78365', + 'snippet/edit-file' => 'c2f074f4f26929c92795a75775af79a6ed6d8430', + 'snippet/no-files' => '671aaa842a4875e5f30082d1ab6feda345fdb94d', '2-mb-file' => 'bf12d25', 'before-create-delete-modify-move' => '845009f', 'between-create-delete-modify-move' => '3f5f443', |
