diff options
36 files changed, 1430 insertions, 458 deletions
diff --git a/CHANGELOG-EE.md b/CHANGELOG-EE.md index c570a5d2fdf..af6a62850ea 100644 --- a/CHANGELOG-EE.md +++ b/CHANGELOG-EE.md @@ -1,5 +1,12 @@ Please view this file on the master branch, on stable branches it's out of date. +## 12.9.3 (2020-04-14) + +### Security (1 change) + +- Fix filename bypass when uploading NuGet packages. + + ## 12.9.2 (2020-03-31) ### Fixed (4 changes) @@ -152,6 +159,13 @@ Please view this file on the master branch, on stable branches it's out of date. - Allow users to be marked as service users. !202680 +## 12.8.9 (2020-04-14) + +### Security (1 change) + +- Fix filename bypass when uploading NuGet packages. + + ## 12.8.7 (2020-03-16) ### Fixed (1 change) @@ -305,6 +319,13 @@ Please view this file on the master branch, on stable branches it's out of date. - Prepare DB structure for GMA forking changes. !22002 +## 12.7.9 (2020-04-14) + +### Security (1 change) + +- Fix filename bypass when uploading NuGet packages. + + ## 12.7.5 ### Fixed (1 change) diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index 5239b90e3c2..57b9fc187c0 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -8.30.0 +8.30.1 @@ -163,7 +163,7 @@ gem 'diffy', '~> 3.3' gem 'diff_match_patch', '~> 0.1.0' # Application server -gem 'rack', '~> 2.0.7' +gem 'rack', '~> 2.0.9' group :unicorn do gem 'unicorn', '~> 5.4.1' diff --git a/Gemfile.lock b/Gemfile.lock index c65e2252ce6..d705bf61e47 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -173,7 +173,7 @@ GEM concord (0.1.5) adamantium (~> 0.2.0) equalizer (~> 0.0.9) - concurrent-ruby (1.1.5) + concurrent-ruby (1.1.6) connection_pool (2.2.2) contracts (0.11.0) cork (0.3.0) @@ -788,7 +788,7 @@ GEM public_suffix (4.0.3) pyu-ruby-sasl (0.0.3.3) raabro (1.1.6) - rack (2.0.7) + rack (2.0.9) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.2.0) @@ -859,17 +859,17 @@ GEM json recursive-open-struct (1.1.0) redis (4.1.3) - redis-actionpack (5.1.0) - actionpack (>= 4.0, < 7) - redis-rack (>= 1, < 3) + redis-actionpack (5.2.0) + actionpack (>= 5, < 7) + redis-rack (>= 2.1.0, < 3) redis-store (>= 1.1.0, < 2) redis-activesupport (5.2.0) activesupport (>= 3, < 7) redis-store (>= 1.3, < 2) redis-namespace (1.6.0) redis (>= 3.0.4) - redis-rack (2.0.6) - rack (>= 1.5, < 3) + redis-rack (2.1.2) + rack (>= 2.0.8, < 3) redis-store (>= 1.2, < 2) redis-rails (5.0.2) redis-actionpack (>= 5.0, < 6) @@ -1331,7 +1331,7 @@ DEPENDENCIES prometheus-client-mmap (~> 0.10.0) pry-byebug (~> 3.5.1) pry-rails (~> 0.3.9) - rack (~> 2.0.7) + rack (~> 2.0.9) rack-attack (~> 6.2.0) rack-cors (~> 1.0.6) rack-oauth2 (~> 1.9.3) diff --git a/app/models/active_session.rb b/app/models/active_session.rb index f37da1b7f59..050155398ab 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -6,31 +6,32 @@ class ActiveSession SESSION_BATCH_SIZE = 200 ALLOWED_NUMBER_OF_ACTIVE_SESSIONS = 100 - attr_writer :session_id - attr_accessor :created_at, :updated_at, :ip_address, :browser, :os, :device_name, :device_type, - :is_impersonated + :is_impersonated, :session_id def current?(session) return false if session_id.nil? || session.id.nil? - session_id == session.id + # Rack v2.0.8+ added private_id, which uses the hash of the + # public_id to avoid timing attacks. + session_id.private_id == session.id.private_id end def human_device_type device_type&.titleize end + # This is not the same as Rack::Session::SessionId#public_id, but we + # need to preserve this for backwards compatibility. def public_id - encrypted_id = Gitlab::CryptoHelper.aes256_gcm_encrypt(session_id) - CGI.escape(encrypted_id) + Gitlab::CryptoHelper.aes256_gcm_encrypt(session_id.public_id) end def self.set(user, request) Gitlab::Redis::SharedState.with do |redis| - session_id = request.session.id + session_id = request.session.id.public_id client = DeviceDetector.new(request.user_agent) timestamp = Time.current @@ -63,32 +64,35 @@ class ActiveSession def self.list(user) Gitlab::Redis::SharedState.with do |redis| - cleaned_up_lookup_entries(redis, user).map do |entry| - # rubocop:disable Security/MarshalLoad - Marshal.load(entry) - # rubocop:enable Security/MarshalLoad + cleaned_up_lookup_entries(redis, user).map do |raw_session| + load_raw_session(raw_session) end end end def self.destroy(user, session_id) + return unless session_id + Gitlab::Redis::SharedState.with do |redis| destroy_sessions(redis, user, [session_id]) end end def self.destroy_with_public_id(user, public_id) - session_id = decrypt_public_id(public_id) - destroy(user, session_id) unless session_id.nil? + decrypted_id = decrypt_public_id(public_id) + + return if decrypted_id.nil? + + session_id = Rack::Session::SessionId.new(decrypted_id) + destroy(user, session_id) end def self.destroy_sessions(redis, user, session_ids) - key_names = session_ids.map {|session_id| key_name(user.id, session_id) } - session_names = session_ids.map {|session_id| "#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id}" } + key_names = session_ids.map { |session_id| key_name(user.id, session_id.public_id) } - redis.srem(lookup_key_name(user.id), session_ids) + redis.srem(lookup_key_name(user.id), session_ids.map(&:public_id)) redis.del(key_names) - redis.del(session_names) + redis.del(rack_session_keys(session_ids)) end def self.cleanup(user) @@ -110,28 +114,65 @@ class ActiveSession sessions_from_ids(session_ids_for_user(user.id)) end + # Lists the relevant session IDs for the user. + # + # Returns an array of Rack::Session::SessionId objects def self.session_ids_for_user(user_id) Gitlab::Redis::SharedState.with do |redis| - redis.smembers(lookup_key_name(user_id)) + session_ids = redis.smembers(lookup_key_name(user_id)) + session_ids.map { |id| Rack::Session::SessionId.new(id) } end end + # Lists the ActiveSession objects for the given session IDs. + # + # session_ids - An array of Rack::Session::SessionId objects + # + # Returns an array of ActiveSession objects def self.sessions_from_ids(session_ids) return [] if session_ids.empty? Gitlab::Redis::SharedState.with do |redis| - session_keys = session_ids.map { |session_id| "#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id}" } + session_keys = rack_session_keys(session_ids) session_keys.each_slice(SESSION_BATCH_SIZE).flat_map do |session_keys_batch| redis.mget(session_keys_batch).compact.map do |raw_session| - # rubocop:disable Security/MarshalLoad - Marshal.load(raw_session) - # rubocop:enable Security/MarshalLoad + load_raw_session(raw_session) end end end end + # Deserializes an ActiveSession object from Redis. + # + # raw_session - Raw bytes from Redis + # + # Returns an ActiveSession object + def self.load_raw_session(raw_session) + # rubocop:disable Security/MarshalLoad + session = Marshal.load(raw_session) + # rubocop:enable Security/MarshalLoad + + # Older ActiveSession models serialize `session_id` as strings, To + # avoid breaking older sessions, we keep backwards compatibility + # with older Redis keys and initiate Rack::Session::SessionId here. + session.session_id = Rack::Session::SessionId.new(session.session_id) if session.try(:session_id).is_a?(String) + session + end + + def self.rack_session_keys(session_ids) + session_ids.each_with_object([]) do |session_id, arr| + # This is a redis-rack implementation detail + # (https://github.com/redis-store/redis-rack/blob/master/lib/rack/session/redis.rb#L88) + # + # We need to delete session keys based on the legacy public key name + # and the newer private ID keys, but there's no well-defined interface + # so we have to do it directly. + arr << "#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id.public_id}" + arr << "#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id.private_id}" + end + end + def self.raw_active_session_entries(redis, session_ids, user_id) return [] if session_ids.empty? @@ -146,7 +187,7 @@ class ActiveSession entry_keys = raw_active_session_entries(redis, session_ids, user_id) entry_keys.compact.map do |raw_session| - Marshal.load(raw_session) # rubocop:disable Security/MarshalLoad + load_raw_session(raw_session) end end @@ -159,10 +200,13 @@ class ActiveSession sessions = active_session_entries(session_ids, user.id, redis) sessions.sort_by! {|session| session.updated_at }.reverse! destroyable_sessions = sessions.drop(ALLOWED_NUMBER_OF_ACTIVE_SESSIONS) - destroyable_session_ids = destroyable_sessions.map { |session| session.send :session_id } # rubocop:disable GitlabSecurity/PublicSend + destroyable_session_ids = destroyable_sessions.map { |session| session.session_id } destroy_sessions(redis, user, destroyable_session_ids) if destroyable_session_ids.any? end + # Cleans up the lookup set by removing any session IDs that are no longer present. + # + # Returns an array of marshalled ActiveModel objects that are still active. def self.cleaned_up_lookup_entries(redis, user) session_ids = session_ids_for_user(user.id) entries = raw_active_session_entries(redis, session_ids, user.id) @@ -181,13 +225,8 @@ class ActiveSession end private_class_method def self.decrypt_public_id(public_id) - decoded_id = CGI.unescape(public_id) - Gitlab::CryptoHelper.aes256_gcm_decrypt(decoded_id) + Gitlab::CryptoHelper.aes256_gcm_decrypt(public_id) rescue nil end - - private - - attr_reader :session_id end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index fb65432024e..9c887fc87f3 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -2,17 +2,19 @@ # rubocop:disable Rails/ActiveRecordAliases class WikiPage + include Gitlab::Utils::StrongMemoize + PageChangedError = Class.new(StandardError) PageRenameError = Class.new(StandardError) - - MAX_TITLE_BYTES = 245 - MAX_DIRECTORY_BYTES = 255 + FrontMatterTooLong = Class.new(StandardError) include ActiveModel::Validations include ActiveModel::Conversion include StaticModel extend ActiveModel::Naming + delegate :content, :front_matter, to: :parsed_content + def self.primary_key 'slug' end @@ -114,8 +116,7 @@ class WikiPage @attributes[:title] = new_title end - # The raw content of this page. - def content + def raw_content @attributes[:content] ||= @page&.text_data end @@ -238,7 +239,7 @@ class WikiPage save do wiki.update_page( @page, - content: content, + content: raw_content, format: format, message: attrs[:message], title: title @@ -281,8 +282,10 @@ class WikiPage # Updates the current @attributes hash by merging a hash of params def update_attributes(attrs) attrs[:title] = process_title(attrs[:title]) if attrs[:title].present? + update_front_matter(attrs) attrs.slice!(:content, :format, :message, :title) + clear_memoization(:parsed_content) if attrs.has_key?(:content) @attributes.merge!(attrs) end @@ -293,6 +296,28 @@ class WikiPage private + def serialize_front_matter(hash) + return '' unless hash.present? + + YAML.dump(hash.transform_keys(&:to_s)) + "---\n" + end + + def update_front_matter(attrs) + return unless Gitlab::WikiPages::FrontMatterParser.enabled?(project) + return unless attrs.has_key?(:front_matter) + + fm_yaml = serialize_front_matter(attrs[:front_matter]) + raise FrontMatterTooLong if fm_yaml.size > Gitlab::WikiPages::FrontMatterParser::MAX_FRONT_MATTER_LENGTH + + attrs[:content] = fm_yaml + (attrs[:content].presence || content) + end + + def parsed_content + strong_memoize(:parsed_content) do + Gitlab::WikiPages::FrontMatterParser.new(raw_content, project).parse + end + end + # Process and format the title based on the user input. def process_title(title) return if title.blank? @@ -339,14 +364,16 @@ class WikiPage def validate_path_limits *dirnames, title = @attributes[:title].split('/') - if title && title.bytesize > MAX_TITLE_BYTES - errors.add(:title, _("exceeds the limit of %{bytes} bytes") % { bytes: MAX_TITLE_BYTES }) + if title && title.bytesize > Gitlab::WikiPages::MAX_TITLE_BYTES + errors.add(:title, _("exceeds the limit of %{bytes} bytes") % { + bytes: Gitlab::WikiPages::MAX_TITLE_BYTES + }) end - invalid_dirnames = dirnames.select { |d| d.bytesize > MAX_DIRECTORY_BYTES } + invalid_dirnames = dirnames.select { |d| d.bytesize > Gitlab::WikiPages::MAX_DIRECTORY_BYTES } invalid_dirnames.each do |dirname| errors.add(:title, _('exceeds the limit of %{bytes} bytes for directory name "%{dirname}"') % { - bytes: MAX_DIRECTORY_BYTES, + bytes: Gitlab::WikiPages::MAX_DIRECTORY_BYTES, dirname: dirname }) end diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index c9c6b54a791..9437eb9eede 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -29,7 +29,15 @@ module Groups group.chat_team&.remove_mattermost_team(current_user) + user_ids_for_project_authorizations_refresh = group.user_ids_for_project_authorizations + group.destroy + + UserProjectAccessChangedService + .new(user_ids_for_project_authorizations_refresh) + .execute(blocking: true) + + group end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/changelogs/unreleased/ajk-wiki-frontmatter.yml b/changelogs/unreleased/ajk-wiki-frontmatter.yml new file mode 100644 index 00000000000..2b3e908d23c --- /dev/null +++ b/changelogs/unreleased/ajk-wiki-frontmatter.yml @@ -0,0 +1,5 @@ +--- +title: Read metadata from Wiki front-matter +merge_request: 27706 +author: +type: added diff --git a/db/post_migrate/20200204113225_schedule_recalculate_project_authorizations_third_run.rb b/db/post_migrate/20200204113225_schedule_recalculate_project_authorizations_third_run.rb new file mode 100644 index 00000000000..47b22b4800a --- /dev/null +++ b/db/post_migrate/20200204113225_schedule_recalculate_project_authorizations_third_run.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class ScheduleRecalculateProjectAuthorizationsThirdRun < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'RecalculateProjectAuthorizationsWithMinMaxUserId' + BATCH_SIZE = 2_500 + DELAY_INTERVAL = 2.minutes.to_i + + disable_ddl_transaction! + + class User < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'users' + end + + def up + say "Scheduling #{MIGRATION} jobs" + + queue_background_migration_jobs_by_range_at_intervals(User, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + end +end diff --git a/db/structure.sql b/db/structure.sql index f924d69fd75..f478fc3d709 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12947,6 +12947,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200204070729 20200204113223 20200204113224 +20200204113225 20200204131054 20200204131831 20200205143231 diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 21839c7393e..3451219c267 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -2,6 +2,7 @@ Akismet Alertmanager Algolia Ansible +Anthos API approvers Artifactory @@ -118,6 +119,8 @@ hardcode hardcoded hardcodes Helm +Heroku +Herokuish HipChat hostname hostnames @@ -133,6 +136,7 @@ initializers interdependencies interdependency Irker +Istio jasmine-jquery JavaScript Jaeger @@ -180,6 +184,7 @@ misconfiguring mitigations mockup mockups +ModSecurity nameserver nameservers namespace @@ -326,6 +331,10 @@ unchecking unchecks uncomment uncommented +unencode +unencoded +unencoder +unencodes unencrypted Unicorn unindexed diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 91908d6c5e4..27e968c0a40 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -22,10 +22,9 @@ Pipelines comprise: Jobs are executed by [Runners](../runners/README.md). Multiple jobs in the same stage are executed in parallel, if there are enough concurrent runners. -If all the jobs in a stage: +If *all* jobs in a stage succeed, the pipeline moves on to the next stage. -- Succeed, the pipeline moves on to the next stage. -- Fail, the next stage is not (usually) executed and the pipeline ends early. +If *any* job in a stage fails, the next stage is not (usually) executed and the pipeline ends early. In general, pipelines are executed automatically and require no intervention once created. However, there are also times when you can manually interact with a pipeline. @@ -46,6 +45,10 @@ you may need to enable pipeline triggering in your project's Pipelines can be configured in many different ways: +- [Basic pipelines](pipeline_architectures.md#basic-pipelines) run everything in each stage concurrently, + followed by the next stage. +- [Directed Acyclic Graph Pipeline (DAG) pipelines](../directed_acyclic_graph/index.md) are based on relationships + between jobs and can run more quickly than basic pipelines. - [Multi-project pipelines](../multi_project_pipelines.md) combine pipelines for different projects together. - [Parent-Child pipelines](../parent_child_pipelines.md) break down complex pipelines into one parent pipeline that can trigger multiple child sub-pipelines, which all diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 28abd73ef97..d373c3212c3 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -1,25 +1,26 @@ # Customizing Auto DevOps -While Auto DevOps provides great defaults to get you started, you can customize -almost everything to fit your needs; from custom [buildpacks](#custom-buildpacks), -to [`Dockerfile`s](#custom-dockerfile), [Helm charts](#custom-helm-chart), or -even copying the complete [CI/CD configuration](#customizing-gitlab-ciyml) -into your project to enable staging and canary deployments, and more. +While [Auto DevOps](index.md) provides great defaults to get you started, you can customize +almost everything to fit your needs. Auto DevOps offers everything from custom +[buildpacks](#custom-buildpacks), to [`Dockerfiles](#custom-dockerfile), and +[Helm charts](#custom-helm-chart). You can even copy the complete +[CI/CD configuration](#customizing-gitlab-ciyml) into your project to enable +staging and canary deployments, and more. ## Custom buildpacks If the automatic buildpack detection fails for your project, or if you want to -use a custom buildpack, you can override the buildpack(s) using a project variable +use a custom buildpack, you can override the buildpack using a project variable or a `.buildpacks` file in your project: - **Project variable** - Create a project variable `BUILDPACK_URL` with the URL of the buildpack to use. -- **`.buildpacks` file** - Add a file in your project's repo called `.buildpacks` +- **`.buildpacks` file** - Add a file in your project's repository called `.buildpacks`, and add the URL of the buildpack to use on a line in the file. If you want to - use multiple buildpacks, you can enter them in, one on each line. + use multiple buildpacks, enter one buildpack per line. The buildpack URL can point to either a Git repository URL or a tarball URL. -For Git repositories, it is possible to point to a specific Git reference (for example, +For Git repositories, you can point to a specific Git reference (such as commit SHA, tag name, or branch name) by appending `#<ref>` to the Git repository URL. For example: @@ -29,11 +30,10 @@ For example: ### Multiple buildpacks -Using multiple buildpacks isn't fully supported by Auto DevOps because, when using the `.buildpacks` -file, Auto Test will not work. - -The buildpack [heroku-buildpack-multi](https://github.com/heroku/heroku-buildpack-multi/), -which is used under the hood to parse the `.buildpacks` file, doesn't provide the necessary commands +Using multiple buildpacks is not fully supported by Auto DevOps, because Auto Test +won't work when using the `.buildpacks` file. The buildpack +[heroku-buildpack-multi](https://github.com/heroku/heroku-buildpack-multi/), used +in the backend to parse the `.buildpacks` file, does not provide the necessary commands `bin/test-compile` and `bin/test`. If your goal is to use only a single custom buildpack, you should provide the project variable @@ -41,18 +41,16 @@ If your goal is to use only a single custom buildpack, you should provide the pr ## Custom `Dockerfile` -If your project has a `Dockerfile` in the root of the project repo, Auto DevOps -will build a Docker image based on the Dockerfile rather than using buildpacks. +If your project has a `Dockerfile` in the root of the project repository, Auto DevOps +builds a Docker image based on the Dockerfile, rather than using buildpacks. This can be much faster and result in smaller images, especially if your Dockerfile is based on [Alpine](https://hub.docker.com/_/alpine/). ## Passing arguments to `docker build` Arguments can be passed to the `docker build` command using the -`AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` project variable. - -For example, to build a Docker image based on based on the `ruby:alpine` -instead of the default `ruby:latest`: +`AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` project variable. For example, to build a +Docker image based on based on the `ruby:alpine` instead of the default `ruby:latest`: 1. Set `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` to `--build-arg=RUBY_VERSION=alpine`. 1. Add the following to a custom `Dockerfile`: @@ -65,25 +63,28 @@ instead of the default `ruby:latest`: ``` NOTE: **Note:** -Passing in complex values (newlines and spaces, for example) will likely -cause escaping issues due to the way this argument is used in Auto DevOps. -Consider using Base64 encoding of such values to avoid this problem. +Use Base64 encoding if you need to pass complex values, such as newlines and +spaces. Left unencoded, complex values like these can cause escaping issues +due to how Auto DevOps uses the arguments. CAUTION: **Warning:** Avoid passing secrets as Docker build arguments if possible, as they may be persisted in your image. See -[this discussion](https://github.com/moby/moby/issues/13490) for details. +[this discussion of best practices with secrets](https://github.com/moby/moby/issues/13490) for details. ## Passing secrets to `docker build` > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25514) in GitLab 12.3, but available in versions 11.9 and above. -CI environment variables can be passed as [build -secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to the `docker build` command by listing them comma separated by name in the -`AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` variable. For example, in order to forward the variables `CI_COMMIT_SHA` and `CI_ENVIRONMENT_NAME`, one would set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`. +CI environment variables can be passed as +[build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to the `docker build` command by listing them +by name, comma-separated, in the `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` +variable. For example, to forward the variables `CI_COMMIT_SHA` and `CI_ENVIRONMENT_NAME`, +set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`. -Unlike build arguments, these are not persisted by Docker in the final image -(though you can still persist them yourself, so **be careful**). +CAUTION: **Caution:** +Unlike build arguments, these variables are not persisted by Docker in the final image, +though you can still persist them yourself. In projects: @@ -91,7 +92,7 @@ In projects: variables. - With a `Dockerfile`, the following is required: - 1. Activate the experimental `Dockerfile` syntax by adding the following + 1. Activate the experimental `Dockerfile` syntax by adding the following code to the top of the file: ```dockerfile @@ -114,30 +115,33 @@ feature to use the `--secret` flag. Auto DevOps uses [Helm](https://helm.sh/) to deploy your application to Kubernetes. You can override the Helm chart used by bundling up a chart into your project -repo or by specifying a project variable: +repository or by specifying a project variable: - **Bundled chart** - If your project has a `./chart` directory with a `Chart.yaml` - file in it, Auto DevOps will detect the chart and use it instead of the [default - one](https://gitlab.com/gitlab-org/charts/auto-deploy-app). - This can be a great way to control exactly how your application is deployed. + file in it, Auto DevOps will detect the chart and use it instead of the + [default chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app), enabling + you to control exactly how your application is deployed. - **Project variable** - Create a [project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables) - `AUTO_DEVOPS_CHART` with the URL of a custom chart to use or create two project variables `AUTO_DEVOPS_CHART_REPOSITORY` with the URL of a custom chart repository and `AUTO_DEVOPS_CHART` with the path to the chart. + `AUTO_DEVOPS_CHART` with the URL of a custom chart to use, or create two project + variables: `AUTO_DEVOPS_CHART_REPOSITORY` with the URL of a custom chart repository, + and `AUTO_DEVOPS_CHART` with the path to the chart. ## Customize values for Helm Chart > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades. -You can override the default values in the `values.yaml` file in the [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). -This can be achieved by either: +You can override the default values in the `values.yaml` file in the +[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) by either: -- Adding a file named `.gitlab/auto-deploy-values.yaml` to your repository. It will - be automatically used if found. -- Adding a file with a different name or path to the repository, and set the - `HELM_UPGRADE_VALUES_FILE` [environment variable](#environment-variables) with the path and name. +- Adding a file named `.gitlab/auto-deploy-values.yaml` to your repository, which is + automatically used, if found. +- Adding a file with a different name or path to the repository, and setting the + `HELM_UPGRADE_VALUES_FILE` [environment variable](#environment-variables) with + the path and name. NOTE: **Note:** -For GitLab 12.5 and earlier, the `HELM_UPGRADE_EXTRA_ARGS` environment variable can be used to override the default chart values. -To do so, set `HELM_UPGRADE_EXTRA_ARGS` to `--values my-values.yaml`. +For GitLab 12.5 and earlier, use the `HELM_UPGRADE_EXTRA_ARGS` environment variable +to override the default chart values by setting `HELM_UPGRADE_EXTRA_ARGS` to `--values <my-values.yaml>`. ## Custom Helm chart per environment @@ -146,34 +150,34 @@ to the desired environment. See [Limiting environment scopes of variables](../.. ## Customizing `.gitlab-ci.yml` -Auto DevOps is completely customizable because the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml): +Auto DevOps is completely customizable because the +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +is just an implementation of a [`.gitlab-ci.yml`](../../ci/yaml/README.md) file, +and uses only features available to any implementation of `.gitlab-ci.yml`. -- Is just an implementation of a [`.gitlab-ci.yml`](../../ci/yaml/README.md) file. -- Uses only features available to any implementation of `.gitlab-ci.yml`. - -If you want to modify the CI/CD pipeline used by Auto DevOps, you can [`include` -the template](../../ci/yaml/README.md#includetemplate) and customize as -needed. To do this, add a `.gitlab-ci.yml` file to the root of your repository +To modify the CI/CD pipeline used by Auto DevOps, +[`include` the template](../../ci/yaml/README.md#includetemplate), and customize +it as needed by adding a `.gitlab-ci.yml` file to the root of your repository containing the following: -```yml +```yaml include: - template: Auto-DevOps.gitlab-ci.yml ``` -Then add any extra changes you want. Your additions will be merged with the -[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) using the behaviour described for -[`include`](../../ci/yaml/README.md#include). +Add your changes, and your additions will be merged with the +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +using the behavior described for [`include`](../../ci/yaml/README.md#include). -It is also possible to copy and paste the contents of the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) -into your project and edit this as needed. You may prefer to do it -that way if you want to specifically remove any part of it. +If you need to specifically remove a part of the file, you can also copy and paste the contents of the +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +into your project and edit it as needed. ## Customizing the Kubernetes namespace > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. -For **non**-GitLab-managed clusters, the namespace can be customized using +For clusters not managed by GitLab, you can customize the namespace in `.gitlab-ci.yml` by specifying [`environment:kubernetes:namespace`](../../ci/environments.md#configuring-kubernetes-deployments). For example, the following configuration overrides the namespace used for @@ -212,15 +216,14 @@ include: - template: Jobs/Build.gitlab-ci.yml ``` -Consult the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs. +See the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs. ## PostgreSQL database support -In order to support applications that require a database, +To support applications requiring a database, [PostgreSQL](https://www.postgresql.org/) is provisioned by default. The credentials to access the database are preconfigured, but can be customized by setting the associated -[variables](#environment-variables). These credentials can be used for defining a -`DATABASE_URL` of the format: +[variables](#environment-variables). You can use these credentials to define a `DATABASE_URL`: ```yaml postgres://user:password@postgres-host:postgres-port/postgres-database @@ -230,7 +233,7 @@ postgres://user:password@postgres-host:postgres-port/postgres-database CAUTION: **Deprecation** The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned -PostgreSQL currently defaults to `1`. This is scheduled to change to `2` in +PostgreSQL currently defaults to `1`. This value is scheduled to change to `2` in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499). The version of the chart used to provision PostgreSQL: @@ -250,19 +253,18 @@ To use the new PostgreSQL: ### Using external PostgreSQL database providers While Auto DevOps provides out-of-the-box support for a PostgreSQL container for -production environments, for some use-cases it may not be sufficiently secure or -resilient and you may wish to use an external managed provider for PostgreSQL. -For example, AWS Relational Database Service. +production environments, for some use cases, it may not be sufficiently secure or +resilient, and you may want to use an external managed provider (such as +AWS Relational Database Service) for PostgreSQL. -You will need to define environment-scoped variables for `POSTGRES_ENABLED` and `DATABASE_URL` in your project's CI/CD settings. - -To achieve this: +You must define environment-scoped variables for `POSTGRES_ENABLED` and +`DATABASE_URL` in your project's CI/CD settings: 1. Disable the built-in PostgreSQL installation for the required environments using scoped [environment variables](../../ci/environments.md#scoping-environments-with-specs). For this use case, it's likely that only `production` will need to be added to this - list as the builtin PostgreSQL setup for Review Apps and staging will be sufficient - as a high availability setup is not required. + list. The built-in PostgreSQL setup for Review Apps and staging is sufficient, + because a high availability setup is not required.  @@ -273,14 +275,14 @@ To achieve this: postgres://user:password@postgres-host:postgres-port/postgres-database ``` -You will need to ensure that your Kubernetes cluster has network access to wherever +You must ensure that your Kubernetes cluster has network access to wherever PostgreSQL is hosted. ## Environment variables The following variables can be used for setting up the Auto DevOps domain, providing a custom Helm chart, or scaling your application. PostgreSQL can -also be customized, and you can easily use a [custom buildpack](#custom-buildpacks). +also be customized, and you can use a [custom buildpack](#custom-buildpacks). ### Build and deployment @@ -292,34 +294,34 @@ applications. | `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | | `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | | `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). | -| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes will not prevent word splitting. [More details](#passing-arguments-to-docker-build). | +| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes won't prevent word splitting. [More details](#passing-arguments-to-docker-build). | | `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. | | `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). | | `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. | | `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | From GitLab 11.11, used to set the name of the Helm repository. Defaults to `gitlab`. | | `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | | `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | -| `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [Modsecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [Modsecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. | +| `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [ModSecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [ModSecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. | | `BUILDPACK_URL` | Buildpack's full URL. Can point to either [a Git repository URL or a tarball URL](#custom-buildpacks). | | `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). | | `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. | | `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. | | `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. | | `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. | -| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes will not prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. | +| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes won't prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. | | `INCREMENTAL_ROLLOUT_MODE` | From GitLab 11.4, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | | `K8S_SECRET_*` | From GitLab 11.7, any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) will be made available by Auto DevOps as environment variables to the deployed application. | | `KUBE_INGRESS_BASE_DOMAIN` | From GitLab 11.8, can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/index.md#base-domain) for more information. | | `PRODUCTION_REPLICAS` | Number of replicas to deploy in the production environment. Takes precedence over `REPLICAS` and defaults to 1. For zero downtime upgrades, set to 2 or greater. | | `REPLICAS` | Number of replicas to deploy. Defaults to 1. | | `ROLLOUT_RESOURCE_TYPE` | From GitLab 11.9, allows specification of the resource type being deployed when using a custom Helm chart. Default value is `deployment`. | -| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it doesn't support all resource types, for example, `cronjob`. | +| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it does not support all resource types, for example, `cronjob`. | | `STAGING_ENABLED` | From GitLab 10.8, used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | TIP: **Tip:** Set up the replica variables using a [project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables) -and scale your application by just redeploying it! +and scale your application by only redeploying it. CAUTION: **Caution:** You should *not* scale your application using Kubernetes directly. This can @@ -334,7 +336,7 @@ The following table lists variables related to the database. |-----------------------------------------|------------------------------------| | `DB_INITIALIZE` | From GitLab 11.4, used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. | | `DB_MIGRATE` | From GitLab 11.4, used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. | -| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `"true"`. Set to `false` to disable the automatic deployment of PostgreSQL. | +| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `true`. Set to `false` to disable the automatic deployment of PostgreSQL. | | `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. | | `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. | | `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/README.md#predefined-environment-variables). Set it to use a custom database name. | @@ -354,35 +356,34 @@ The following table lists variables used to disable jobs. | **Variable** | **Description** | |-----------------------------------------|------------------------------------| -| `CODE_QUALITY_DISABLED` | From GitLab 11.0, used to disable the `codequality` job. If the variable is present, the job will not be created. | -| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `sast:container` job. If the variable is present, the job will not be created. | -| `DAST_DISABLED` | From GitLab 11.0, used to disable the `dast` job. If the variable is present, the job will not be created. | -| `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `dependency_scanning` job. If the variable is present, the job will not be created. | -| `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, used to disable the `license_management` job. If the variable is present, the job will not be created. | -| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the `performance` job. If the variable is present, the job will not be created. | -| `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. | -| `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job will not be created. | -| `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job will not be created. | +| `CODE_QUALITY_DISABLED` | From GitLab 11.0, used to disable the `codequality` job. If the variable is present, the job won't be created. | +| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `sast:container` job. If the variable is present, the job won't be created. | +| `DAST_DISABLED` | From GitLab 11.0, used to disable the `dast` job. If the variable is present, the job won't be created. | +| `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `dependency_scanning` job. If the variable is present, the job won't be created. | +| `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, used to disable the `license_management` job. If the variable is present, the job won't be created. | +| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the `performance` job. If the variable is present, the job won't be created. | +| `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs won't be created. | +| `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job won't be created. | +| `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job won't be created. | ### Application secret variables > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/49056) in GitLab 11.7. -Some applications need to define secret variables that are -accessible by the deployed application. Auto DevOps detects variables where the key starts with -`K8S_SECRET_` and make these prefixed variables available to the -deployed application, as environment variables. +Some applications need to define secret variables that are accessible by the deployed +application. Auto DevOps detects variables starting with `K8S_SECRET_`, and makes +these prefixed variables available to the deployed application as environment variables. To configure your application variables: -1. Go to your project's **Settings > CI/CD**, then expand the section - called **Variables**. +1. Go to your project's **{settings}** **Settings > CI/CD**, then expand the + **Variables** section. -1. Create a CI Variable, ensuring the key is prefixed with +1. Create a CI/CD variable, ensuring the key is prefixed with `K8S_SECRET_`. For example, you can create a variable with key `K8S_SECRET_RAILS_MASTER_KEY`. -1. Run an Auto Devops pipeline either by manually creating a new +1. Run an Auto DevOps pipeline, either by manually creating a new pipeline or by pushing a code change to GitLab. Auto DevOps pipelines will take your application secret variables to @@ -394,6 +395,7 @@ example above, you can see the secret below containing the ```shell $ kubectl get secret production-secret -n minimal-ruby-app-54 -o yaml + apiVersion: v1 data: RAILS_MASTER_KEY: MTIzNC10ZXN0 @@ -408,40 +410,37 @@ metadata: type: Opaque ``` -Environment variables are generally considered immutable in a Kubernetes -pod. Therefore, if you update an application secret without changing any -code then manually create a new pipeline, you will find that any running -application pods will not have the updated secrets. In this case, you -can either push a code update to GitLab to force the Kubernetes -Deployment to recreate pods or manually delete running pods to -cause Kubernetes to create new pods with updated secrets. +Environment variables are generally considered immutable in a Kubernetes pod. +If you update an application secret without changing any code, then manually +create a new pipeline, you will find any running application pods won't have +the updated secrets. To update the secrets, either: + +- Push a code update to GitLab to force the Kubernetes deployment to recreate pods. +- Manually delete running pods to cause Kubernetes to create new pods with updated + secrets. NOTE: **Note:** -Variables with multiline values are not currently supported due to +Variables with multi-line values are not currently supported due to limitations with the current Auto DevOps scripting environment. ### Advanced replica variables setup Apart from the two replica-related variables for production mentioned above, -you can also use others for different environments. +you can also use other variables for different environments. -There's a very specific mapping between Kubernetes' label named `track`, -GitLab CI/CD environment names, and the replicas environment variable. -The general rule is: `TRACK_ENV_REPLICAS`. Where: +The Kubernetes' label named `track`, GitLab CI/CD environment names, and the +replicas environment variable are combined into the format `TRACK_ENV_REPLICAS`, +enabling you to define your own variables for scaling the pod's replicas: - `TRACK`: The capitalized value of the `track` [Kubernetes label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) - in the Helm Chart app definition. If not set, it will not be taken into account + in the Helm Chart app definition. If not set, it won't be taken into account to the variable name. -- `ENV`: The capitalized environment name of the deploy job that is set in +- `ENV`: The capitalized environment name of the deploy job, set in `.gitlab-ci.yml`. -That way, you can define your own `TRACK_ENV_REPLICAS` variables with which -you will be able to scale the pod's replicas easily. - -In the example below, the environment's name is `qa` and it deploys the track -`foo` which would result in looking for the `FOO_QA_REPLICAS` environment -variable: +In the example below, the environment's name is `qa`, and it deploys the track +`foo`, which results in an environment variable named `FOO_QA_REPLICAS`: ```yaml QA testing: @@ -452,8 +451,7 @@ QA testing: - deploy foo ``` -The track `foo` being referenced would also need to be defined in the -application's Helm chart, like: +The track `foo` being referenced must also be defined in the application's Helm chart, like: ```yaml replicaCount: 1 @@ -482,30 +480,29 @@ service: TIP: **Tip:** You can also set this inside your [project's settings](index.md#deployment-strategy). -The normal behavior of Auto DevOps is to use Continuous Deployment, pushing +The normal behavior of Auto DevOps is to use continuous deployment, pushing automatically to the `production` environment every time a new pipeline is run on the default branch. However, there are cases where you might want to use a -staging environment and deploy to production manually. For this scenario, the +staging environment, and deploy to production manually. For this scenario, the `STAGING_ENABLED` environment variable was introduced. -If `STAGING_ENABLED` is defined in your project (e.g., set `STAGING_ENABLED` to -`1` as a CI/CD variable), then the application will be automatically deployed -to a `staging` environment, and a `production_manual` job will be created for +If you define `STAGING_ENABLED`, such as setting `STAGING_ENABLED` to +`1` as a CI/CD variable, then GitLab automatically deploys the application +to a `staging` environment, and creates a `production_manual` job for you when you're ready to manually deploy to production. ### Deploy policy for canary environments **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-ci-yml/-/merge_requests/171) in GitLab 11.0. -A [canary environment](../../user/project/canary_deployments.md) can be used -before any changes are deployed to production. +You can use a [canary environment](../../user/project/canary_deployments.md) before +deploying any changes to production. -If `CANARY_ENABLED` is defined in your project (e.g., set `CANARY_ENABLED` to -`1` as a CI/CD variable) then two manual jobs will be created: +If you define `CANARY_ENABLED` in your project, such as setting `CANARY_ENABLED` to +`1` as a CI/CD variable, then two manual jobs are created: -- `canary` which will deploy the application to the canary environment -- `production_manual` which is to be used by you when you're ready to manually - deploy to production. +- `canary` - Deploys the application to the canary environment. +- `production_manual` - Manually deploys the application to production. ### Incremental rollout to production **(PREMIUM)** @@ -514,10 +511,9 @@ If `CANARY_ENABLED` is defined in your project (e.g., set `CANARY_ENABLED` to TIP: **Tip:** You can also set this inside your [project's settings](index.md#deployment-strategy). -When you have a new version of your app to deploy in production, you may want -to use an incremental rollout to replace just a few pods with the latest code. -This will allow you to first check how the app is behaving, and later manually -increasing the rollout up to 100%. +When you're ready to deploy a new version of your app to production, you may want +to use an incremental rollout to replace just a few pods with the latest code to +check how the application is behaving before manually increasing the rollout up to 100%. If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead of the standard `production` job, 4 different @@ -529,14 +525,14 @@ will be created: 1. `rollout 50%` 1. `rollout 100%` -The percentage is based on the `REPLICAS` variable and defines the number of -pods you want to have for your deployment. If you say `10`, and then you run -the `10%` rollout job, there will be `1` new pod + `9` old ones. +The percentage is based on the `REPLICAS` variable, and defines the number of +pods you want to have for your deployment. If the value is `10`, and you run the +`10%` rollout job, there will be `1` new pod + `9` old ones. -To start a job, click on the play icon next to the job's name. You are not +To start a job, click the play icon (**{play}**) next to the job's name. You're not required to go from `10%` to `100%`, you can jump to whatever job you want. You can also scale down by running a lower percentage job, just before hitting -`100%`. Once you get to `100%`, you cannot scale down, and you'd have to roll +`100%`. Once you get to `100%`, you can't scale down, and you'd have to roll back by redeploying the old version using the [rollback button](../../ci/environments.md#retrying-and-rolling-back) in the environment page. @@ -561,9 +557,9 @@ With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`  CAUTION: **Caution:** -Before GitLab 11.4 this feature was enabled by the presence of the -`INCREMENTAL_ROLLOUT_ENABLED` environment variable. -This configuration is deprecated and will be removed in the future. +Before GitLab 11.4, the presence of the `INCREMENTAL_ROLLOUT_ENABLED` environment +variable enabled this feature. This configuration is deprecated, and will be +removed in the future. ### Timed incremental rollout to production **(PREMIUM)** @@ -577,8 +573,10 @@ This configuration is based on Everything behaves the same way, except: -- It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`. -- Instead of the standard `production` job, the following jobs are created with a 5 minute delay between each : +- To enable it, set the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`. +- Instead of the standard `production` job, the following jobs are created with + a 5 minute delay between each: + 1. `timed rollout 10%` 1. `timed rollout 25%` 1. `timed rollout 50%` @@ -586,15 +584,15 @@ Everything behaves the same way, except: ## Auto DevOps banner -The following Auto DevOps banner will show for maintainers+ on new projects when Auto DevOps is not -enabled: +The following Auto DevOps banner displays for users with Maintainer or greater +permissions on new projects when Auto DevOps is not enabled:  The banner can be disabled for: -- A user when they dismiss it themselves. -- A project by explicitly [disabling Auto DevOps](index.md#enablingdisabling-auto-devops). +- A user, when they dismiss it themselves. +- A project, by explicitly [disabling Auto DevOps](index.md#enablingdisabling-auto-devops). - An entire GitLab instance: - By an administrator running the following in a Rails console: diff --git a/doc/topics/autodevops/img/guide_environments_metrics_v12_3.png b/doc/topics/autodevops/img/guide_environments_metrics_v12_3.png Binary files differdeleted file mode 100644 index 47bf3fda53a..00000000000 --- a/doc/topics/autodevops/img/guide_environments_metrics_v12_3.png +++ /dev/null diff --git a/doc/topics/autodevops/img/guide_project_landing_page_v12_10.png b/doc/topics/autodevops/img/guide_project_landing_page_v12_10.png Binary files differnew file mode 100644 index 00000000000..ec646c50a20 --- /dev/null +++ b/doc/topics/autodevops/img/guide_project_landing_page_v12_10.png diff --git a/doc/topics/autodevops/img/guide_project_landing_page_v12_3.png b/doc/topics/autodevops/img/guide_project_landing_page_v12_3.png Binary files differdeleted file mode 100644 index 7eb2ee3692f..00000000000 --- a/doc/topics/autodevops/img/guide_project_landing_page_v12_3.png +++ /dev/null diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 043bb0dfcfa..7ed6625bea3 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -418,12 +418,23 @@ spec: ## Troubleshooting -- Auto Build and Auto Test may fail in detecting your language/framework. There - may be no buildpack for your application, or your application may be missing the - key files the buildpack is looking for. For example, for Ruby applications, you must - have a `Gemfile` to be properly detected, even though it is possible to write a - Ruby app without a `Gemfile`. Try specifying a [custom - buildpack](customize.md#custom-buildpacks). +- Auto Build and Auto Test may fail to detect your language or framework with the + following error: + + ```plaintext + Step 5/11 : RUN /bin/herokuish buildpack build + ---> Running in eb468cd46085 + -----> Unable to select a buildpack + The command '/bin/sh -c /bin/herokuish buildpack build' returned a non-zero code: 1 + ``` + + The following are possible reasons: + + - Your application may be missing the key files the buildpack is looking for. For + example, for Ruby applications you must have a `Gemfile` to be properly detected, + even though it is possible to write a Ruby app without a `Gemfile`. + - There may be no buildpack for your application. Try specifying a + [custom buildpack](customize.md#custom-buildpacks). - Auto Test may fail because of a mismatch between testing frameworks. In this case, you may need to customize your `.gitlab-ci.yml` with your test commands. - Auto Deploy will fail if GitLab can not create a Kubernetes namespace and diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 493e3b727a8..3fb4aa64f3f 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -1,42 +1,45 @@ # Getting started with Auto DevOps -This is a step-by-step guide that will help you use [Auto DevOps](index.md) to +This step-by-step guide will help you use [Auto DevOps](index.md) to deploy a project hosted on GitLab.com to Google Kubernetes Engine. -We will use GitLab's native Kubernetes integration, so you will not need +You will use GitLab's native Kubernetes integration, so you won't need to create a Kubernetes cluster manually using the Google Cloud Platform console. -We will create and deploy a simple application that we create from a GitLab template. +You will create and deploy a simple application that you create from a GitLab template. These instructions will also work for a self-managed GitLab instance; you'll just need to ensure your own [Runners are configured](../../ci/runners/README.md) and [Google OAuth is enabled](../../integration/google.md). -## Configuring your Google account +## Configure your Google account Before creating and connecting your Kubernetes cluster to your GitLab project, -you need a Google Cloud Platform account. If you don't already have one, -sign up at <https://console.cloud.google.com>. You'll need to either sign in with an existing -Google account (for example, one that you use to access Gmail, Drive, etc.) or create a new one. +you need a [Google Cloud Platform account](https://console.cloud.google.com). +Sign in with an existing Google account, such as the one you use to access Gmail +or Google Drive, or create a new one. -1. Follow the steps as outlined in the ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) - in order for the required APIs and related services to be enabled. -1. Make sure you have created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account). +1. Follow the steps described in the ["Before you begin" section](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) + of the Kubernetes Engine docs to enable the required APIs and related services. +1. Ensure you've created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) + with Google Cloud Platform. TIP: **Tip:** Every new Google Cloud Platform (GCP) account receives [$300 in credit](https://console.cloud.google.com/freetrial), -and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's -Google Kubernetes Engine Integration. All you have to do is [follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) and apply for credit. +and in partnership with Google, GitLab is able to offer an additional $200 for new +GCP accounts to get started with GitLab's Google Kubernetes Engine Integration. +[Follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) +and apply for credit. -## Creating a new project from a template +## Create a new project from a template We will use one of GitLab's project templates to get started. As the name suggests, those projects provide a bare-bones application built on some well-known frameworks. -1. In GitLab, click the plus icon (**+**) at the top of the navigation bar and select +1. In GitLab, click the plus icon (**{plus-square}**) at the top of the navigation bar, and select **New project**. -1. Go to the **Create from template** tab where you can choose among a Ruby on +1. Go to the **Create from template** tab, where you can choose among a Ruby on Rails, Spring, or NodeJS Express project. - We'll use the Ruby on Rails template. + For this tutorial, use the Ruby on Rails template.  @@ -48,196 +51,215 @@ those projects provide a bare-bones application built on some well-known framewo 1. Click **Create project**. -Now that the project is created, the next step is to create the Kubernetes cluster -under which this application will be deployed. +Now that you've created a project, you'll next create the Kubernetes cluster +to deploy this project to. -## Creating a Kubernetes cluster from within GitLab +## Create a Kubernetes cluster from within GitLab -1. On the project's landing page, click the button labeled **Add Kubernetes cluster** - (note that this option is also available when you navigate to **Operations > Kubernetes**). +1. On your project's landing page, click **Add Kubernetes cluster** + (note that this option is also available when you navigate to **{cloud-gear}** **Operations > Kubernetes**). -  +  -1. One the **Create new cluster on GKE** tab, click "Sign in with Google". +1. On the **Add a Kubernetes cluster integration** page, click the **Create new cluster** tab, + then click **Google GKE**.  -1. Connect with your Google account and press **Allow** when asked (this will - be shown only the first time you connect GitLab with your Google account). - -  - -1. The last step is to provide the cluster details. Give it a name, leave the - environment scope as is, and choose the GCP project under which the cluster - will be created. (Per the instructions when you - [configured your Google account](#configuring-your-google-account), a project - should have already been created for you.) Next, choose the - [region/zone](https://cloud.google.com/compute/docs/regions-zones/) under which the - cluster will be created, enter the number of nodes you want it to have, and - finally choose the [machine type](https://cloud.google.com/compute/docs/machine-types). - -  - -1. Once ready, click **Create Kubernetes cluster**. +1. Connect with your Google account, and click **Allow** to allow access to your + Google account. (This authorization request is only displayed the first time + you connect GitLab with your Google account.) + + After authorizing access, the **Add a Kubernetes cluster integration** page + is displayed. + +1. In the **Enter the details for your Kubernetes cluster** section, provide + details about your cluster: + + - **Kubernetes cluster name** + - **Environment scope** - Leave this field unchanged. + - **Google Cloud Platform project** - Select a project. When you + [configured your Google account](#configure-your-google-account), a project + should have already been created for you. + - **Zone** - The [region/zone](https://cloud.google.com/compute/docs/regions-zones/) to + create the cluster in. + - **Number of nodes** + - **Machine type** - For more information about + [machine types](https://cloud.google.com/compute/docs/machine-types), see Google's documentation. + - **Enable Cloud Run for Anthos** - Select this checkbox to use the Cloud Run, + Istio, and HTTP Load Balancing add-ons for this cluster. + - **GitLab-managed cluster** - Select this checkbox to + [allow GitLab to manage namespace and service accounts](../..//user/project/clusters/index.md#gitlab-managed-clusters) for this cluster. + +1. Click **Create Kubernetes cluster**. After a couple of minutes, the cluster will be created. You can also see its status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). -The next step is to install some applications on your cluster that are needed +Next, you will install some applications on your cluster that are needed to take full advantage of Auto DevOps. -## Installing Helm, Ingress, and Prometheus +## Install the package manager -GitLab's Kubernetes integration comes with some +After creating your Kubernetes cluster, GitLab's Kubernetes integration provides [pre-defined applications](../../user/project/clusters/index.md#installing-applications) -for you to install. - - - -The first one to install is Helm Tiller, a package manager for Kubernetes, which -is needed in order to install the rest of the applications. Go ahead and click -its **Install** button. +for you to install. To install them, you must next install Helm Tiller, the +Kubernetes package manager for Kubernetes, to enable the installation of other applications. -Once it's installed, the other applications that rely on it will each have their **Install** -button enabled. For this guide, we need Ingress and Prometheus. Ingress provides -load balancing, SSL termination, and name-based virtual hosting, using NGINX behind -the scenes. Prometheus is an open-source monitoring and alerting system that we'll -use to supervise the deployed application. We will not install GitLab Runner as -we'll use the shared Runners that GitLab.com provides. +Next to **Helm Tiller**, click **Install**. -After the Ingress is installed, wait a few seconds and copy the IP address that -is displayed in order to add in your base **Domain** at the top of the page. For -the purpose of this guide, we will use the one suggested by GitLab. Once you have -filled in the domain, click **Save changes**. - - + -## Enabling Auto DevOps (optional) +After installation completes, the page reloads, and you can install other +applications. -Starting with GitLab 11.3, Auto DevOps is enabled by default. However, it is possible to disable -Auto DevOps at both the instance-level (for self-managed instances) and also at the group-level. -Follow these steps if Auto DevOps has been manually disabled. +## Install Ingress and Prometheus -1. First, navigate to **Settings > CI/CD > Auto DevOps**. -1. Select **Default to Auto DevOps pipeline**. -1. Lastly, let's select the [continuous deployment strategy](index.md#deployment-strategy) - which will automatically deploy the application to production once the pipeline - successfully runs on the `master` branch. -1. Click **Save changes**. +After installing **Helm Tiller**, you can install other applications that rely on it, +including Ingress and Prometheus, which we will install in this quick start guide: -  +- Ingress - Provides load balancing, SSL termination, and name-based virtual hosting, + using NGINX behind the scenes. +- Prometheus - An open-source monitoring and alerting system used to supervise the + deployed application. -Once you complete all the above and save your changes, a new pipeline is -automatically created. To view the pipeline, go to **CI/CD > Pipelines**. +NOTE: **Note:** +We won't install GitLab Runner in this quick start guide, as this guide uses the +shared Runners provided by GitLab.com. - +To install the applications: -In the next section we'll break down the pipeline and explain what each job does. +- Click the **Install** button for **Ingress**. +- When the **Ingress Endpoint** is displayed, copy the IP address. +- Add your **Base domain**. For this guide, we will use the domain suggested by GitLab. +- Click **Save changes**. -## Deploying the application + -By now you should see the pipeline running, but what is it running exactly? +## Enable Auto DevOps (optional) -To navigate inside the pipeline, click its status badge. (Its status should be "running"). -The pipeline is split into 4 stages, each running a couple of jobs. +While Auto DevOps is enabled by default, Auto DevOps can be disabled at both +the instance level (for self-managed instances) and the group level. Complete +these steps to enable Auto DevOps if it's disabled: - +1. Navigate to **{settings}** **Settings > CI/CD > Auto DevOps**, and click **Expand**. +1. Select **Default to Auto DevOps pipeline** to display more options. +1. In **Deployment strategy**, select your desired [continuous deployment strategy](index.md#deployment-strategy) + to deploy the application to production after the pipeline successfully runs on the `master` branch. +1. Click **Save changes**. -In the **build** stage, the application is built into a Docker image and then -uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](stages.md#auto-build)). +  -In the **test** stage, GitLab runs various checks on the application: +After you save your changes, GitLab creates a new pipeline. To view it, go to +**{rocket}** **CI/CD > Pipelines**. -- The `test` job runs unit and integration tests by detecting the language and - framework ([Auto Test](stages.md#auto-test)) -- The `code_quality` job checks the code quality and is allowed to fail - ([Auto Code Quality](stages.md#auto-code-quality-starter)) **(STARTER)** -- The `container_scanning` job checks the Docker container if it has any - vulnerabilities and is allowed to fail ([Auto Container Scanning](stages.md#auto-container-scanning-ultimate)) -- The `dependency_scanning` job checks if the application has any dependencies - susceptible to vulnerabilities and is allowed to fail ([Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate)) **(ULTIMATE)** -- The `sast` job runs static analysis on the current code to check for potential - security issues and is allowed to fail([Auto SAST](stages.md#auto-sast-ultimate)) **(ULTIMATE)** -- The `license_management` job searches the application's dependencies to determine each of their - licenses and is allowed to fail ([Auto License Compliance](stages.md#auto-license-compliance-ultimate)) **(ULTIMATE)** + -NOTE: **Note:** -As you might have noticed, all jobs except `test` are allowed to fail in the -test stage. +In the next section, we explain what each job does in the pipeline. -The **production** stage is run after the tests and checks finish, and it automatically -deploys the application in Kubernetes ([Auto Deploy](stages.md#auto-deploy)). +## Deploy the application -Lastly, in the **performance** stage, some performance tests will run -on the deployed application -([Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium)). **(PREMIUM)** +When your pipeline runs, what is it doing? ---- +To view the jobs in the pipeline, click the pipeline's status badge. The +**{status_running}** icon displays when pipeline jobs are running, and updates +without refreshing the page to **{status_success}** (for success) or +**{status_failed}** (for failure) when the jobs complete. -The URL for the deployed application can be found under the **Environments** -page where you can also monitor your application. Let's explore that. +The jobs are separated into stages: -### Monitoring + -Now that the application is successfully deployed, let's navigate to its -website. First, go to **Operations > Environments**. +- **Build** - The application builds a Docker image and uploads it to your project's + [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](stages.md#auto-build)). +- **Test** - GitLab runs various checks on the application: + + - The `test` job runs unit and integration tests by detecting the language and + framework ([Auto Test](stages.md#auto-test)) + - The `code_quality` job checks the code quality and is allowed to fail + ([Auto Code Quality](stages.md#auto-code-quality-starter)) **(STARTER)** + - The `container_scanning` job checks the Docker container if it has any + vulnerabilities and is allowed to fail ([Auto Container Scanning](stages.md#auto-container-scanning-ultimate)) + - The `dependency_scanning` job checks if the application has any dependencies + susceptible to vulnerabilities and is allowed to fail + ([Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate)) **(ULTIMATE)** + - The `sast` job runs static analysis on the current code to check for potential + security issues and is allowed to fail([Auto SAST](stages.md#auto-sast-ultimate)) **(ULTIMATE)** + - The `license_management` job searches the application's dependencies to determine each of their + licenses and is allowed to fail + ([Auto License Compliance](stages.md#auto-license-compliance-ultimate)) **(ULTIMATE)** + + NOTE: **Note:** + All jobs except `test` are allowed to fail in the test stage. + +- **Production** - After the tests and checks finish, the application deploys in + Kubernetes ([Auto Deploy](stages.md#auto-deploy)). + +- **Performance** - Performance tests are run on the deployed application + ([Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium)). **(PREMIUM)** + +After running a pipeline, you should view your deployed website and learn how +to monitor it. + +### Monitor your project + +After successfully deploying your application, you can view its website and check +on its health on the **Environments** page by navigating to +**{cloud-gear}** **Operations > Environments**. This page displays details about +the deployed applications, and the right-hand column displays icons that link +you to common environment tasks:  -In **Environments** you can see some details about the deployed -applications. In the rightmost column for the production environment, you can make use of the three icons: - -- The first icon will open the URL of the application that is deployed in - production. It's a very simple page, but the important part is that it works! -- The next icon, with the small graph, will take you to the metrics page where - Prometheus collects data about the Kubernetes cluster and how the application - affects it (in terms of memory/CPU usage, latency, etc.). - -  - -- The third icon is the [web terminal](../../ci/environments.md#web-terminals) - and it will open a terminal session right inside the container where the - application is running. - -Right below, there is the -[Deploy Board](../../user/project/deploy_boards.md). -The squares represent pods in your Kubernetes cluster that are associated with -the given environment. Hovering above each square you can see the state of a -deployment and clicking a square will take you to the pod's logs page. +- **{external-link}** **Open live environment** - Opens the URL of the application deployed in production +- **{chart}** **Monitoring** - Opens the metrics page where Prometheus collects data + about the Kubernetes cluster and how the application + affects it in terms of memory usage, CPU usage, and latency +- **{play}** **{angle-down}** **Deploy to** - Displays a list of environments you can deploy to +- **{terminal}** **Terminal** - Opens a [web terminal](../../ci/environments.md#web-terminals) + session inside the container where the application is running +- **{repeat}** **Re-deploy to environment** +- **{stop}** **Stop environment** + +GitLab displays the [Deploy Board](../../user/project/deploy_boards.md) below the +environment's information, with squares representing pods in your +Kubernetes cluster, color-coded to show their status. Hovering over a square on +the deploy board displays the state of the deployment, and clicking the square +takes you to the pod's logs page. TIP: **Tip:** -There is only one pod hosting the application at the moment, but you can add +The example shows only one pod hosting the application at the moment, but you can add more pods by defining the [`REPLICAS` variable](customize.md#environment-variables) -under **Settings > CI/CD > Environment variables**. +in **{settings}** **Settings > CI/CD > Environment variables**. -### Working with branches +### Work with branches Following the [GitLab flow](../gitlab_flow.md#working-with-feature-branches), -let's create a feature branch that will add some content to the application. +you should next create a feature branch to add content to your application: -Under your repository, navigate to the following file: `app/views/welcome/index.html.erb`. -By now, it should only contain a paragraph: `<p>You're on Rails!</p>`, so let's -start adding content. Let's use GitLab's [Web IDE](../../user/project/web_ide/index.md) to make the change. Once -you're on the Web IDE, make the following change: +1. In your project's repository, navigate to the following file: `app/views/welcome/index.html.erb`. + This file should only contain a paragraph: `<p>You're on Rails!</p>`. +1. Open the GitLab [Web IDE](../../user/project/web_ide/index.md) to make the change. +1. Edit the file so it contains: -```html -<p>You're on Rails! Powered by GitLab Auto DevOps.</p> -``` + ```html + <p>You're on Rails! Powered by GitLab Auto DevOps.</p> + ``` -Stage the file, add a commit message, and create a new branch and a merge request -by clicking **Commit**. +1. Stage the file. Add a commit message, then create a new branch and a merge request + by clicking **Commit**. - +  -Once you submit the merge request, you'll see the pipeline running. This will -run all the jobs as [described previously](#deploying-the-application), as well as +After submitting the merge request, GitLab runs your pipeline, and all the jobs +in it, as [described previously](#deploy-the-application), in addition to a few more that run only on branches other than `master`.  -After a few minutes you'll notice that there was a failure in a test. -This means there's a test that was 'broken' by our change. -Navigating into the `test` job that failed, you can see what the broken test is: +After a few minutes you'll notice a test failed, which means a test was +'broken' by your change. Click on the failed `test` job to see more information +about it: ```plaintext Failure: @@ -249,30 +271,31 @@ Expected 0 to be >= 1. bin/rails test test/controllers/welcome_controller_test.rb:4 ``` -Let's fix that: +To fix the broken test: -1. Back to the merge request, click the **Open in Web IDE** button. -1. Find the `test/controllers/welcome_controller_test.rb` file and open it. +1. Return to the **Overview** page for your merge request, and click **Open in Web IDE**. +1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` + file, and click it to open it. 1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` 1. Click **Commit**. -1. On your left, under "Unstaged changes", click the little checkmark icon +1. In the left-hand column, under **Unstaged changes**, click the checkmark icon to stage the changes. -1. Write a commit message and click **Commit**. +1. Write a commit message, and click **Commit**. -Now, if you go back to the merge request you should not only see the test passing, but -also the application deployed as a [review app](stages.md#auto-review-apps). You -can visit it by following clicking the **View app** button. You will see -the changes that we previously made. +Return to the **Overview** page of your merge request, and you should not only +see the test passing, but also the application deployed as a +[review application](stages.md#auto-review-apps). You can visit it by clicking +the **View app** button to see your changes deployed.  -Once you merge the merge request, the pipeline will run on the `master` branch, -and the application will be eventually deployed straight to production. +After merging the merge request, GitLab runs the pipeline on the `master` branch, +and then deploys the application to production. ## Conclusion -After implementing this project, you should now have a solid understanding of the basics of Auto DevOps. -We started from building and testing to deploying and monitoring an application +After implementing this project, you should have a solid understanding of the basics of Auto DevOps. +You started from building and testing, to deploying and monitoring an application all within GitLab. Despite its automatic nature, Auto DevOps can also be configured and customized to fit your workflow. Here are some helpful resources for further reading: diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md index 1d3380230af..6483a56e7f7 100644 --- a/doc/topics/web_application_firewall/quick_start_guide.md +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -58,7 +58,7 @@ under which this application will be deployed. 1. On the project's landing page, click **Add Kubernetes cluster** (note that this option is also available when you navigate to **Operations > Kubernetes**). -  +  1. On the **Create new cluster on GKE** tab, click **Sign in with Google**. diff --git a/doc/user/application_security/dast/img/dast_urls_scanned_v12_10.png b/doc/user/application_security/dast/img/dast_urls_scanned_v12_10.png Binary files differnew file mode 100644 index 00000000000..c15a2da513c --- /dev/null +++ b/doc/user/application_security/dast/img/dast_urls_scanned_v12_10.png diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index c9c7129dd7b..6f51aaf4931 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -524,6 +524,12 @@ does not make any unsolicited requests including checking for updates. The DAST job can emit various reports. +### List of URLs scanned + +When DAST completes scanning, the merge request page states the number of URLs that were scanned. Click **View details** to view the web console output which includes the list of scanned URLs. + + + ### JSON CAUTION: **Caution:** diff --git a/lib/api/runner.rb b/lib/api/runner.rb index f97e28de628..9095aba7340 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -251,21 +251,14 @@ module API end params do requires :id, type: Integer, desc: %q(Job's ID) + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware)) optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: Ci::JobArtifact.file_types.keys optional :artifact_format, type: String, desc: %q(The format of artifact), default: 'zip', values: Ci::JobArtifact.file_formats.keys - optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) - optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) - optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) - optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse)) - optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse)) - optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) - optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) - optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse)) - optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse)) + optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)) end post '/:id/artifacts' do not_allowed! unless Gitlab.config.artifacts.enabled @@ -274,10 +267,9 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path) - metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path) + artifacts = params[:file] + metadata = params[:metadata] - bad_request!('Missing artifacts file!') unless artifacts file_too_large! unless artifacts.size < max_artifacts_size(job) result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata) diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index 544231adea4..5900e762244 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -3,28 +3,11 @@ module Banzai module Filter class FrontMatterFilter < HTML::Pipeline::Filter - DELIM_LANG = { - '---' => 'yaml', - '+++' => 'toml', - ';;;' => 'json' - }.freeze - - DELIM = Regexp.union(DELIM_LANG.keys) - - PATTERN = %r{ - \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line - \s* - ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier) - \s* - ^(?<front_matter>.*?) # front matter (not greedy) - \s* - ^\k<delim> # closing front matter marker - \s* - }mx.freeze - def call - html.sub(PATTERN) do |_match| - lang = $~[:lang].presence || DELIM_LANG[$~[:delim]] + lang_mapping = Gitlab::FrontMatter::DELIM_LANG + + html.sub(Gitlab::FrontMatter::PATTERN) do |_match| + lang = $~[:lang].presence || lang_mapping[$~[:delim]] ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n") end diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb new file mode 100644 index 00000000000..7612bd36aca --- /dev/null +++ b/lib/gitlab/front_matter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module FrontMatter + DELIM_LANG = { + '---' => 'yaml', + '+++' => 'toml', + ';;;' => 'json' + }.freeze + + DELIM = Regexp.union(DELIM_LANG.keys) + + PATTERN = %r{ + \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line + \s* + ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier) + \s* + ^(?<front_matter>.*?) # front matter block content (not greedy) + \s* + ^(\k<delim> | \.{3}) # closing front matter marker + \s* + }mx.freeze + end +end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index cb4213d23a4..c82c05e7319 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -107,6 +107,7 @@ module Gitlab [ ::FileUploader.root, Gitlab.config.uploads.storage_path, + JobArtifactUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] end @@ -125,6 +126,8 @@ module Gitlab Handler.new(env, message).with_open_files do @app.call(env) end + rescue UploadedFile::InvalidPathError => e + [400, { 'Content-Type' => 'text/plain' }, e.message] end end end diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb new file mode 100644 index 00000000000..47f9aa1117f --- /dev/null +++ b/lib/gitlab/wiki_pages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module WikiPages + # Many common file systems have a limit of 255 bytes for file and + # directory names, and while Git and GitLab both support paths exceeding + # those limits, the presence of them makes it impossible for users on + # those file systems to checkout a wiki repository locally. + + # To avoid this situation, we enforce these limits when editing pages + # through the GitLab web interface and API: + MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension + MAX_DIRECTORY_BYTES = 255 + end +end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb new file mode 100644 index 00000000000..45dc6cf7fd1 --- /dev/null +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module WikiPages + class FrontMatterParser + FEATURE_FLAG = :wiki_front_matter + + # We limit the maximum length of text we are prepared to parse as YAML, to + # avoid exploitations and attempts to consume memory and CPU. We allow for: + # - a title line + # - a "slugs:" line + # - and up to 50 slugs + # + # This limit does not take comments into account. + MAX_SLUGS = 50 + SLUG_LINE_LENGTH = (4 + Gitlab::WikiPages::MAX_DIRECTORY_BYTES + 1 + Gitlab::WikiPages::MAX_TITLE_BYTES) + MAX_FRONT_MATTER_LENGTH = (8 + Gitlab::WikiPages::MAX_TITLE_BYTES) + 7 + (SLUG_LINE_LENGTH * MAX_SLUGS) + + ParseError = Class.new(StandardError) + + class Result + attr_reader :front_matter, :content, :reason, :error + + def initialize(content:, front_matter: {}, reason: nil, error: nil) + @content = content + @front_matter = front_matter.freeze + @reason = reason + @error = error + end + end + + # @param [String] wiki_content + # @param [FeatureGate] feature_gate The scope for feature availability + def initialize(wiki_content, feature_gate) + @wiki_content = wiki_content + @feature_gate = feature_gate + end + + def self.enabled?(gate = nil) + Feature.enabled?(FEATURE_FLAG, gate) + end + + def parse + return empty_result unless enabled? && wiki_content.present? + return empty_result(block.error) unless block.valid? + + Result.new(front_matter: block.data, content: strip_front_matter_block) + rescue ParseError => error + empty_result(:parse_error, error) + end + + class Block + include Gitlab::Utils::StrongMemoize + + def initialize(delim = nil, lang = '', text = nil) + @lang = lang.downcase.presence || Gitlab::FrontMatter::DELIM_LANG[delim] + @text = text + end + + def data + @data ||= YAML.safe_load(text, symbolize_names: true) + rescue Psych::DisallowedClass, Psych::SyntaxError => error + raise ParseError, error.message + end + + def valid? + error.nil? + end + + def error + strong_memoize(:error) { no_match? || too_long? || not_yaml? || not_mapping? } + end + + private + + attr_reader :lang, :text + + def no_match? + :no_match if text.nil? + end + + def not_yaml? + :not_yaml if lang != 'yaml' + end + + def too_long? + :too_long if text.size > MAX_FRONT_MATTER_LENGTH + end + + def not_mapping? + :not_mapping unless data.is_a?(Hash) + end + end + + private + + attr_reader :wiki_content, :feature_gate + + def empty_result(reason = nil, error = nil) + Result.new(content: wiki_content, reason: reason, error: error) + end + + def enabled? + self.class.enabled?(feature_gate) + end + + def block + @block ||= parse_front_matter_block + end + + def parse_front_matter_block + wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new + end + + def strip_front_matter_block + wiki_content.gsub(Gitlab::FrontMatter::PATTERN, '') + end + end + end +end diff --git a/spec/lib/gitlab/middleware/multipart_spec.rb b/spec/lib/gitlab/middleware/multipart_spec.rb index 1d3ad3afb56..ec153e25d44 100644 --- a/spec/lib/gitlab/middleware/multipart_spec.rb +++ b/spec/lib/gitlab/middleware/multipart_spec.rb @@ -89,6 +89,17 @@ describe Gitlab::Middleware::Multipart do end end + it 'allows files in the job artifact upload path' do + with_tmp_dir('artifacts') do |dir, env| + expect(JobArtifactUploader).to receive(:workhorse_upload_path).and_return(File.join(dir, 'artifacts')) + expect(app).to receive(:call) do |env| + expect(get_params(env)['file']).to be_a(::UploadedFile) + end + + middleware.call(env) + end + end + it 'allows symlinks for uploads dir' do Tempfile.open('two-levels') do |tempfile| symlinked_dir = '/some/dir/uploads' diff --git a/spec/lib/gitlab/wiki_pages/front_matter_parser_spec.rb b/spec/lib/gitlab/wiki_pages/front_matter_parser_spec.rb new file mode 100644 index 00000000000..c606ba11b9c --- /dev/null +++ b/spec/lib/gitlab/wiki_pages/front_matter_parser_spec.rb @@ -0,0 +1,274 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::WikiPages::FrontMatterParser do + subject(:parser) { described_class.new(raw_content, gate) } + + let(:content) { 'This is the content' } + let(:end_divider) { '---' } + let(:gate) { double('Gate') } + + let(:with_front_matter) do + <<~MD + --- + a: 1 + b: 2 + c: + - foo + - bar + date: I am safe. Not actually a date + #{end_divider} + #{content} + MD + end + + def have_correct_front_matter + include(a: 1, b: 2, c: %w(foo bar)) + end + + describe '#parse' do + subject { parser.parse } + + context 'there is front matter' do + let(:raw_content) { with_front_matter } + + it do + is_expected.to have_attributes( + front_matter: have_correct_front_matter, + content: content + "\n", + error: be_nil + ) + end + end + + context 'there is no content' do + let(:raw_content) { '' } + + it do + is_expected.to have_attributes( + front_matter: {}, + content: raw_content, + error: be_nil + ) + end + end + + context 'there is no front_matter' do + let(:raw_content) { content } + + it { is_expected.to have_attributes(front_matter: be_empty, content: raw_content) } + + it { is_expected.to have_attributes(reason: :no_match) } + end + + context 'the feature flag is disabled' do + let(:raw_content) { with_front_matter } + + before do + stub_feature_flags(Gitlab::WikiPages::FrontMatterParser::FEATURE_FLAG => false) + end + + it { is_expected.to have_attributes(front_matter: be_empty, content: raw_content) } + end + + context 'the feature flag is enabled for the gated object' do + let(:raw_content) { with_front_matter } + + before do + stub_feature_flags(Gitlab::WikiPages::FrontMatterParser::FEATURE_FLAG => false) + stub_feature_flags(Gitlab::WikiPages::FrontMatterParser::FEATURE_FLAG => { + enabled: true, + thing: gate + }) + end + + it do + is_expected.to have_attributes( + front_matter: have_correct_front_matter, + content: content + "\n", + reason: be_nil + ) + end + end + + context 'the end divider is ...' do + let(:end_divider) { '...' } + let(:raw_content) { with_front_matter } + + it { is_expected.to have_attributes(front_matter: have_correct_front_matter) } + end + + context 'the front-matter is not a mapping' do + let(:raw_content) do + <<~MD + --- + - thing one + - thing two + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :not_mapping) } + end + + context 'there is nothing in the front-matter block' do + let(:raw_content) do + <<~MD + --- + --- + My content here + MD + end + + it { is_expected.to have_attributes(reason: :not_mapping) } + end + + context 'there is a string in the YAML block' do + let(:raw_content) do + <<~MD + --- + This is a string + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :not_mapping) } + end + + context 'there is dangerous YAML in the block' do + let(:raw_content) do + <<~MD + --- + date: 2010-02-11 11:02:57 + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :parse_error, error: be_present) } + end + + context 'there is acceptably long YAML in the front-matter block' do + let(:raw_content) do + key = 'title: ' + length = described_class::MAX_FRONT_MATTER_LENGTH - key.size + + <<~MD + --- + title: #{FFaker::Lorem.characters(length)} + --- + #{content} + MD + end + + it { is_expected.to have_attributes(front_matter: include(title: be_present)) } + end + + context 'there is suspiciously long YAML in the front-matter block' do + let(:raw_content) do + <<~MD + --- + title: #{FFaker::Lorem.characters(described_class::MAX_FRONT_MATTER_LENGTH)} + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :too_long) } + end + + context 'TOML front matter' do + let(:raw_content) do + <<~MD + +++ + title = "My title" + +++ + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :not_yaml) } + end + + context 'TOML style fences, advertised as YAML' do + let(:raw_content) do + <<~MD + +++ yaml + title: "My title" + +++ + #{content} + MD + end + + it { is_expected.to have_attributes(front_matter: include(title: 'My title')) } + end + + context 'YAML, advertised as something else' do + let(:raw_content) do + <<~MD + --- toml + title: My title + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :not_yaml) } + end + + context 'there is text content in the YAML block, in comments' do + let(:raw_content) do + <<~MD + --- + # This is YAML + # + # It has comments though. Explaining things + foo: 1 + + ## It has headings + + headings: + + - heading one + - heading two + + # And lists + + lists: + + - and lists + - with things in them + --- + #{content} + MD + end + + it { is_expected.to have_attributes(front_matter: include(foo: 1)) } + end + + context 'there is text content in the YAML block' do + let(:raw_content) do + <<~MD + --- + # This is not YAML + + In fact is looks like markdown + + ## It has headings + + Paragraphs + + - and lists + - with things in them + --- + #{content} + MD + end + + it { is_expected.to have_attributes(reason: :not_mapping) } + end + end +end diff --git a/spec/migrations/schedule_recalculate_project_authorizations_third_run_spec.rb b/spec/migrations/schedule_recalculate_project_authorizations_third_run_spec.rb new file mode 100644 index 00000000000..19ba8984224 --- /dev/null +++ b/spec/migrations/schedule_recalculate_project_authorizations_third_run_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20200204113225_schedule_recalculate_project_authorizations_third_run.rb') + +describe ScheduleRecalculateProjectAuthorizationsThirdRun do + let(:users_table) { table(:users) } + + before do + stub_const("#{described_class}::BATCH_SIZE", 2) + + 1.upto(4) do |i| + users_table.create!(id: i, name: "user#{i}", email: "user#{i}@example.com", projects_limit: 1) + end + end + + it 'schedules background migration' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + expect(described_class::MIGRATION).to be_scheduled_migration(1, 2) + expect(described_class::MIGRATION).to be_scheduled_migration(3, 4) + end + end + end +end diff --git a/spec/models/active_session_spec.rb b/spec/models/active_session_spec.rb index 27a80f93566..6a97d91b3ca 100644 --- a/spec/models/active_session_spec.rb +++ b/spec/models/active_session_spec.rb @@ -9,10 +9,8 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do end end - let(:session) do - double(:session, { id: '6919a6f1bb119dd7396fadc38fd18d0d', - '[]': {} }) - end + let(:rack_session) { Rack::Session::SessionId.new('6919a6f1bb119dd7396fadc38fd18d0d') } + let(:session) { instance_double(ActionDispatch::Request::Session, id: rack_session, '[]': {}) } let(:request) do double(:request, { @@ -25,13 +23,13 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do describe '#current?' do it 'returns true if the active session matches the current session' do - active_session = ActiveSession.new(session_id: '6919a6f1bb119dd7396fadc38fd18d0d') + active_session = ActiveSession.new(session_id: rack_session) expect(active_session.current?(session)).to be true end it 'returns false if the active session does not match the current session' do - active_session = ActiveSession.new(session_id: '59822c7d9fcdfa03725eff41782ad97d') + active_session = ActiveSession.new(session_id: Rack::Session::SessionId.new('59822c7d9fcdfa03725eff41782ad97d')) expect(active_session.current?(session)).to be false end @@ -46,14 +44,12 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do describe '#public_id' do it 'returns an encrypted, url-encoded session id' do - original_session_id = "!*'();:@&\n=+$,/?%abcd#123[4567]8" + original_session_id = Rack::Session::SessionId.new("!*'();:@&\n=+$,/?%abcd#123[4567]8") active_session = ActiveSession.new(session_id: original_session_id) - encrypted_encoded_id = active_session.public_id - - encrypted_id = CGI.unescape(encrypted_encoded_id) + encrypted_id = active_session.public_id derived_session_id = Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_id) - expect(original_session_id).to eq derived_session_id + expect(original_session_id.public_id).to eq derived_session_id end end @@ -104,7 +100,8 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do describe '.list_sessions' do it 'uses the ActiveSession lookup to return original sessions' do Gitlab::Redis::SharedState.with do |redis| - redis.set("session:gitlab:6919a6f1bb119dd7396fadc38fd18d0d", Marshal.dump({ _csrf_token: 'abcd' })) + # Emulate redis-rack: https://github.com/redis-store/redis-rack/blob/c75f7f1a6016ee224e2615017fbfee964f23a837/lib/rack/session/redis.rb#L88 + redis.set("session:gitlab:#{rack_session.private_id}", Marshal.dump({ _csrf_token: 'abcd' })) redis.sadd( "session:lookup:user:gitlab:#{user.id}", @@ -127,17 +124,18 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do redis.sadd("session:lookup:user:gitlab:#{user.id}", session_ids) end - expect(ActiveSession.session_ids_for_user(user.id)).to eq(session_ids) + expect(ActiveSession.session_ids_for_user(user.id).map(&:to_s)).to eq(session_ids) end end describe '.sessions_from_ids' do it 'uses the ActiveSession lookup to return original sessions' do Gitlab::Redis::SharedState.with do |redis| - redis.set("session:gitlab:6919a6f1bb119dd7396fadc38fd18d0d", Marshal.dump({ _csrf_token: 'abcd' })) + # Emulate redis-rack: https://github.com/redis-store/redis-rack/blob/c75f7f1a6016ee224e2615017fbfee964f23a837/lib/rack/session/redis.rb#L88 + redis.set("session:gitlab:#{rack_session.private_id}", Marshal.dump({ _csrf_token: 'abcd' })) end - expect(ActiveSession.sessions_from_ids(['6919a6f1bb119dd7396fadc38fd18d0d'])).to eq [{ _csrf_token: 'abcd' }] + expect(ActiveSession.sessions_from_ids([rack_session])).to eq [{ _csrf_token: 'abcd' }] end it 'avoids a redis lookup for an empty array' do @@ -152,11 +150,12 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do redis = double(:redis) expect(Gitlab::Redis::SharedState).to receive(:with).and_yield(redis) - sessions = %w[session-a session-b] + sessions = %w[session-a session-b session-c session-d] mget_responses = sessions.map { |session| [Marshal.dump(session)]} - expect(redis).to receive(:mget).twice.and_return(*mget_responses) + expect(redis).to receive(:mget).exactly(4).times.and_return(*mget_responses) - expect(ActiveSession.sessions_from_ids([1, 2])).to eql(sessions) + session_ids = [1, 2].map { |id| Rack::Session::SessionId.new(id.to_s) } + expect(ActiveSession.sessions_from_ids(session_ids).map(&:to_s)).to eql(sessions) end end @@ -212,6 +211,12 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do end describe '.destroy' do + it 'gracefully handles a nil session ID' do + expect(described_class).not_to receive(:destroy_sessions) + + ActiveSession.destroy(user, nil) + end + it 'removes the entry associated with the currently killed user session' do Gitlab::Redis::SharedState.with do |redis| redis.set("session:user:gitlab:#{user.id}:6919a6f1bb119dd7396fadc38fd18d0d", '') @@ -244,8 +249,9 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do it 'removes the devise session' do Gitlab::Redis::SharedState.with do |redis| - redis.set("session:user:gitlab:#{user.id}:6919a6f1bb119dd7396fadc38fd18d0d", '') - redis.set("session:gitlab:6919a6f1bb119dd7396fadc38fd18d0d", '') + redis.set("session:user:gitlab:#{user.id}:#{rack_session.public_id}", '') + # Emulate redis-rack: https://github.com/redis-store/redis-rack/blob/c75f7f1a6016ee224e2615017fbfee964f23a837/lib/rack/session/redis.rb#L88 + redis.set("session:gitlab:#{rack_session.private_id}", '') end ActiveSession.destroy(user, request.session.id) @@ -322,7 +328,7 @@ RSpec.describe ActiveSession, :clean_gitlab_redis_shared_state do (1..max_number_of_sessions_plus_two).each do |number| redis.set( "session:user:gitlab:#{user.id}:#{number}", - Marshal.dump(ActiveSession.new(session_id: "#{number}", updated_at: number.days.ago)) + Marshal.dump(ActiveSession.new(session_id: number.to_s, updated_at: number.days.ago)) ) redis.sadd( "session:lookup:user:gitlab:#{user.id}", diff --git a/spec/models/wiki_page_spec.rb b/spec/models/wiki_page_spec.rb index 0a0611bdeb8..718b386b3fd 100644 --- a/spec/models/wiki_page_spec.rb +++ b/spec/models/wiki_page_spec.rb @@ -20,6 +20,17 @@ describe WikiPage do subject { new_page } + def disable_front_matter + stub_feature_flags(Gitlab::WikiPages::FrontMatterParser::FEATURE_FLAG => false) + end + + def enable_front_matter_for_project + stub_feature_flags(Gitlab::WikiPages::FrontMatterParser::FEATURE_FLAG => { + thing: project, + enabled: true + }) + end + describe '.group_by_directory' do context 'when there are no pages' do it 'returns an empty array' do @@ -101,6 +112,119 @@ describe WikiPage do end end + describe '#front_matter' do + let_it_be(:project) { create(:project) } + let(:wiki_page) { create(:wiki_page, project: project, content: content) } + + shared_examples 'a page without front-matter' do + it { expect(wiki_page).to have_attributes(front_matter: {}, content: content) } + end + + shared_examples 'a page with front-matter' do + let(:front_matter) { { title: 'Foo', slugs: %w[slug_a slug_b] } } + + it { expect(wiki_page.front_matter).to eq(front_matter) } + end + + context 'the wiki page has front matter' do + let(:content) do + <<~MD + --- + title: Foo + slugs: + - slug_a + - slug_b + --- + + My actual content + MD + end + + it_behaves_like 'a page with front-matter' + + it 'strips the front matter from the content' do + expect(wiki_page.content.strip).to eq('My actual content') + end + + context 'the feature flag is off' do + before do + disable_front_matter + end + + it_behaves_like 'a page without front-matter' + + context 'but enabled for the project' do + before do + enable_front_matter_for_project + end + + it_behaves_like 'a page with front-matter' + end + end + end + + context 'the wiki page does not have front matter' do + let(:content) { 'My actual content' } + + it_behaves_like 'a page without front-matter' + end + + context 'the wiki page has fenced blocks, but nothing in them' do + let(:content) do + <<~MD + --- + --- + + My actual content + MD + end + + it_behaves_like 'a page without front-matter' + end + + context 'the wiki page has invalid YAML type in fenced blocks' do + let(:content) do + <<~MD + --- + this isn't YAML + --- + + My actual content + MD + end + + it_behaves_like 'a page without front-matter' + end + + context 'the wiki page has a disallowed class in fenced block' do + let(:content) do + <<~MD + --- + date: 2010-02-11 11:02:57 + --- + + My actual content + MD + end + + it_behaves_like 'a page without front-matter' + end + + context 'the wiki page has invalid YAML in fenced block' do + let(:content) do + <<~MD + --- + invalid-use-of-reserved-indicator: @text + --- + + My actual content + MD + end + + it_behaves_like 'a page without front-matter' + end + end + describe '.unhyphenize' do it 'removes hyphens from a name' do name = 'a-name--with-hyphens' @@ -155,8 +279,8 @@ describe WikiPage do end describe '#validate_path_limits' do - let(:max_title) { described_class::MAX_TITLE_BYTES } - let(:max_directory) { described_class::MAX_DIRECTORY_BYTES } + let(:max_title) { Gitlab::WikiPages::MAX_TITLE_BYTES } + let(:max_directory) { Gitlab::WikiPages::MAX_DIRECTORY_BYTES } where(:character) do ['a', 'รค', '๐'] @@ -296,7 +420,7 @@ describe WikiPage do subject.update(content: "new content") page = wiki.find_page(title) - expect(page.content).to eq('new content') + expect([subject.content, page.content]).to all(eq('new content')) end it "returns true" do @@ -333,7 +457,7 @@ describe WikiPage do subject.update(content: new_content) page = wiki.find_page('test page') - expect(page.content).to eq("new content") + expect([subject.content, page.content]).to all(eq("new content")) end it "updates the title of the page" do @@ -342,7 +466,75 @@ describe WikiPage do subject.update(title: new_title) page = wiki.find_page(new_title) - expect(page.title).to eq(new_title) + expect([subject.title, page.title]).to all(eq(new_title)) + end + + describe 'updating front_matter' do + shared_examples 'able to update front-matter' do + it 'updates the wiki-page front-matter' do + title = subject.title + content = subject.content + subject.update(front_matter: { slugs: ['x'] }) + page = wiki.find_page(title) + + expect([subject, page]).to all( + have_attributes( + front_matter: include(slugs: include('x')), + content: content + )) + end + end + + it_behaves_like 'able to update front-matter' + + context 'the front matter is too long' do + let(:new_front_matter) do + { + title: generate(:wiki_page_title), + slugs: Array.new(51).map { FFaker::Lorem.characters(512) } + } + end + + it 'raises an error' do + expect { subject.update(front_matter: new_front_matter) }.to raise_error(described_class::FrontMatterTooLong) + end + end + + context 'the front-matter feature flag is not enabled' do + before do + disable_front_matter + end + + it 'does not update the front-matter' do + content = subject.content + subject.update(front_matter: { slugs: ['x'] }) + + page = wiki.find_page(subject.title) + + expect([subject, page]).to all(have_attributes(front_matter: be_empty, content: content)) + end + + context 'but it is enabled for the project' do + before do + enable_front_matter_for_project + end + + it_behaves_like 'able to update front-matter' + end + end + + it 'updates the wiki-page front-matter and content together' do + title = subject.title + content = 'totally new content' + subject.update(content: content, front_matter: { slugs: ['x'] }) + page = wiki.find_page(title) + + expect([subject, page]).to all( + have_attributes( + front_matter: include(slugs: include('x')), + content: content + )) + end end it "returns true" do diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index a67c1a48ad4..bc2da8a2b9a 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -1422,8 +1422,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do describe 'artifacts' do let(:job) { create(:ci_build, :pending, user: user, project: project, pipeline: pipeline, runner_id: runner.id) } - let(:jwt_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } - let(:headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt_token } } + let(:jwt) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } + let(:headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt } } let(:headers_with_token) { headers.merge(API::Helpers::Runner::JOB_TOKEN_HEADER => job.token) } let(:file_upload) { fixture_file_upload('spec/fixtures/banana_sample.gif', 'image/gif') } let(:file_upload2) { fixture_file_upload('spec/fixtures/dk.png', 'image/gif') } @@ -1703,12 +1703,12 @@ describe API::Runner, :clean_gitlab_redis_shared_state do it 'fails to post artifacts without GitLab-Workhorse' do post api("/jobs/#{job.id}/artifacts"), params: { token: job.token }, headers: {} - expect(response).to have_gitlab_http_status(:forbidden) + expect(response).to have_gitlab_http_status(:bad_request) end end context 'Is missing GitLab Workhorse token headers' do - let(:jwt_token) { JWT.encode({ 'iss' => 'invalid-header' }, Gitlab::Workhorse.secret, 'HS256') } + let(:jwt) { JWT.encode({ 'iss' => 'invalid-header' }, Gitlab::Workhorse.secret, 'HS256') } it 'fails to post artifacts without GitLab-Workhorse' do expect(Gitlab::ErrorTracking).to receive(:track_exception).once @@ -1722,15 +1722,14 @@ describe API::Runner, :clean_gitlab_redis_shared_state do context 'when setting an expire date' do let(:default_artifacts_expire_in) {} let(:post_data) do - { 'file.path' => file_upload.path, - 'file.name' => file_upload.original_filename, - 'expire_in' => expire_in } + { file: file_upload, + expire_in: expire_in } end before do stub_application_setting(default_artifacts_expire_in: default_artifacts_expire_in) - post(api("/jobs/#{job.id}/artifacts"), params: post_data, headers: headers_with_token) + upload_artifacts(file_upload, headers_with_token, post_data) end context 'when an expire_in is given' do @@ -1783,20 +1782,22 @@ describe API::Runner, :clean_gitlab_redis_shared_state do let(:stored_artifacts_size) { job.reload.artifacts_size } let(:stored_artifacts_sha256) { job.reload.job_artifacts_archive.file_sha256 } let(:stored_metadata_sha256) { job.reload.job_artifacts_metadata.file_sha256 } + let(:file_keys) { post_data.keys } + let(:send_rewritten_field) { true } before do - post(api("/jobs/#{job.id}/artifacts"), params: post_data, headers: headers_with_token) + workhorse_finalize_with_multiple_files( + api("/jobs/#{job.id}/artifacts"), + method: :post, + file_keys: file_keys, + params: post_data, + headers: headers_with_token, + send_rewritten_field: send_rewritten_field + ) end context 'when posts data accelerated by workhorse is correct' do - let(:post_data) do - { 'file.path' => artifacts.path, - 'file.name' => artifacts.original_filename, - 'file.sha256' => artifacts_sha256, - 'metadata.path' => metadata.path, - 'metadata.name' => metadata.original_filename, - 'metadata.sha256' => metadata_sha256 } - end + let(:post_data) { { file: artifacts, metadata: metadata } } it 'stores artifacts and artifacts metadata' do expect(response).to have_gitlab_http_status(:created) @@ -1808,9 +1809,30 @@ describe API::Runner, :clean_gitlab_redis_shared_state do end end + context 'with a malicious file.path param' do + let(:post_data) { {} } + let(:tmp_file) { Tempfile.new('crafted.file.path') } + let(:url) { "/jobs/#{job.id}/artifacts?file.path=#{tmp_file.path}" } + + it 'rejects the request' do + expect(response).to have_gitlab_http_status(:bad_request) + expect(stored_artifacts_size).to be_nil + end + end + + context 'when workhorse header is missing' do + let(:post_data) { { file: artifacts, metadata: metadata } } + let(:send_rewritten_field) { false } + + it 'rejects the request' do + expect(response).to have_gitlab_http_status(:bad_request) + expect(stored_artifacts_size).to be_nil + end + end + context 'when there is no artifacts file in post data' do let(:post_data) do - { 'metadata' => metadata } + { metadata: metadata } end it 'is expected to respond with bad request' do @@ -2053,7 +2075,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do method: :post, file_key: :file, params: params.merge(file: file), - headers: headers + headers: headers, + send_rewritten_field: true ) end end diff --git a/spec/services/groups/destroy_service_spec.rb b/spec/services/groups/destroy_service_spec.rb index a45c7cdffa6..bf639153b99 100644 --- a/spec/services/groups/destroy_service_spec.rb +++ b/spec/services/groups/destroy_service_spec.rb @@ -133,4 +133,55 @@ describe Groups::DestroyService do end end end + + describe 'authorization updates', :sidekiq_inline do + context 'shared groups' do + let!(:shared_group) { create(:group, :private) } + let!(:shared_group_child) { create(:group, :private, parent: shared_group) } + + let!(:project) { create(:project, group: shared_group) } + let!(:project_child) { create(:project, group: shared_group_child) } + + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + group.refresh_members_authorized_projects + end + + it 'updates project authorization' do + expect(user.can?(:read_project, project)).to eq(true) + expect(user.can?(:read_project, project_child)).to eq(true) + + destroy_group(group, user, false) + + expect(user.can?(:read_project, project)).to eq(false) + expect(user.can?(:read_project, project_child)).to eq(false) + end + end + + context 'shared groups in the same group hierarchy' do + let!(:subgroup) { create(:group, :private, parent: group) } + let!(:subgroup_user) { create(:user) } + + before do + subgroup.add_user(subgroup_user, Gitlab::Access::MAINTAINER) + + create(:group_group_link, shared_group: group, shared_with_group: subgroup) + subgroup.refresh_members_authorized_projects + end + + context 'group is deleted' do + it 'updates project authorization' do + expect { destroy_group(group, user, false) }.to( + change { subgroup_user.can?(:read_project, project) }.from(true).to(false)) + end + end + + context 'subgroup is deleted' do + it 'updates project authorization' do + expect { destroy_group(subgroup, user, false) }.to( + change { subgroup_user.can?(:read_project, project) }.from(true).to(false)) + end + end + end + end end diff --git a/spec/support/helpers/workhorse_helpers.rb b/spec/support/helpers/workhorse_helpers.rb index de232da3c8c..53b36b3dd45 100644 --- a/spec/support/helpers/workhorse_helpers.rb +++ b/spec/support/helpers/workhorse_helpers.rb @@ -33,22 +33,36 @@ module WorkhorseHelpers # workhorse_finalize will transform file_key inside params as if it was the finalize call of an inline object storage upload. # note that based on the content of the params it can simulate a disc acceleration or an object storage upload def workhorse_finalize(url, method: :post, file_key:, params:, headers: {}, send_rewritten_field: false) - workhorse_request_with_file(method, url, - file_key: file_key, - params: params, - extra_headers: headers, - send_rewritten_field: send_rewritten_field + workhorse_finalize_with_multiple_files(url, method: method, file_keys: file_key, params: params, headers: headers, send_rewritten_field: send_rewritten_field) + end + + def workhorse_finalize_with_multiple_files(url, method: :post, file_keys:, params:, headers: {}, send_rewritten_field: false) + workhorse_request_with_multiple_files(method, url, + file_keys: file_keys, + params: params, + extra_headers: headers, + send_rewritten_field: send_rewritten_field ) end def workhorse_request_with_file(method, url, file_key:, params:, env: {}, extra_headers: {}, send_rewritten_field:) + workhorse_request_with_multiple_files(method, url, file_keys: file_key, params: params, env: env, extra_headers: extra_headers, send_rewritten_field: send_rewritten_field) + end + + def workhorse_request_with_multiple_files(method, url, file_keys:, params:, env: {}, extra_headers: {}, send_rewritten_field:) workhorse_params = params.dup - file = workhorse_params.delete(file_key) - workhorse_params = workhorse_disk_accelerated_file_params(file_key, file).merge(workhorse_params) + file_keys = Array(file_keys) + rewritten_fields = {} + + file_keys.each do |key| + file = workhorse_params.delete(key) + rewritten_fields[key] = file.path if file + workhorse_params = workhorse_disk_accelerated_file_params(key, file).merge(workhorse_params) + end headers = if send_rewritten_field - workhorse_rewritten_fields_header(file_key => file.path) + workhorse_rewritten_fields_header(rewritten_fields) else {} end @@ -75,7 +89,11 @@ module WorkhorseHelpers "#{key}.name" => file.original_filename, "#{key}.size" => file.size }.tap do |params| - params["#{key}.path"] = file.path if file.path + if file.path + params["#{key}.path"] = file.path + params["#{key}.sha256"] = Digest::SHA256.file(file.path).hexdigest + end + params["#{key}.remote_id"] = file.remote_id if file.respond_to?(:remote_id) && file.remote_id.present? end end diff --git a/spec/support/rails/test_case_patch.rb b/spec/support/rails/test_case_patch.rb new file mode 100644 index 00000000000..161e1ef2a4c --- /dev/null +++ b/spec/support/rails/test_case_patch.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +# +# This file pulls in the changes in https://github.com/rails/rails/pull/38063 +# to fix controller specs updated with the latest Rack versions. +# +# This file should be removed after that change ships. It is not +# present in Rails 6.0.2.2. +module ActionController + class TestRequest < ActionDispatch::TestRequest #:nodoc: + def self.new_session + TestSessionPatched.new + end + end + + # Methods #destroy and #load! are overridden to avoid calling methods on the + # @store object, which does not exist for the TestSession class. + class TestSessionPatched < Rack::Session::Abstract::PersistedSecure::SecureSessionHash #:nodoc: + DEFAULT_OPTIONS = Rack::Session::Abstract::Persisted::DEFAULT_OPTIONS + + def initialize(session = {}) + super(nil, nil) + @id = Rack::Session::SessionId.new(SecureRandom.hex(16)) + @data = stringify_keys(session) + @loaded = true + end + + def exists? + true + end + + def keys + @data.keys + end + + def values + @data.values + end + + def destroy + clear + end + + def fetch(key, *args, &block) + @data.fetch(key.to_s, *args, &block) + end + + private + + def load! + @id + end + end +end |
