diff options
72 files changed, 746 insertions, 341 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index 3c619b2e534..fdc8f54e9c9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -867,7 +867,7 @@ GEM activesupport (>= 4.2) spring-commands-rspec (1.0.4) spring (>= 0.9.1) - sprockets (3.7.1) + sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.2.1) diff --git a/app/assets/stylesheets/bootstrap_migration.scss b/app/assets/stylesheets/bootstrap_migration.scss index 0d8e867f41d..ba1f0a975a9 100644 --- a/app/assets/stylesheets/bootstrap_migration.scss +++ b/app/assets/stylesheets/bootstrap_migration.scss @@ -173,6 +173,12 @@ table { display: none; } +h3.popover-header { + // Default bootstrap popovers use <h3> + // which we default to having a top margin + margin-top: 0; +} + // Add to .label so that old system notes that are saved to the db // will still receive the correct styling .badge, diff --git a/app/assets/stylesheets/pages/boards.scss b/app/assets/stylesheets/pages/boards.scss index 7c1d1626f1c..750d2c8b990 100644 --- a/app/assets/stylesheets/pages/boards.scss +++ b/app/assets/stylesheets/pages/boards.scss @@ -290,10 +290,6 @@ &.is-active, &.is-active .board-card-assignee:hover a { background-color: $row-hover; - - &:first-child:not(:only-child) { - box-shadow: -10px 0 10px 1px $row-hover; - } } .badge { diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 56312f801fb..21cc6dfdd16 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -27,7 +27,7 @@ class ApplicationController < ActionController::Base after_action :set_page_title_header, if: -> { request.format == :json } - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true helper_method :can? helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled? diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 16abf7bab7e..e54f372344d 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -1,5 +1,5 @@ class HealthController < ActionController::Base - protect_from_forgery with: :exception, except: :storage_check + protect_from_forgery with: :exception, except: :storage_check, prepend: true include RequiresWhitelistedMonitoringClient CHECKS = [ diff --git a/app/controllers/metrics_controller.rb b/app/controllers/metrics_controller.rb index 33b682d2859..0400ffcfee5 100644 --- a/app/controllers/metrics_controller.rb +++ b/app/controllers/metrics_controller.rb @@ -1,7 +1,7 @@ class MetricsController < ActionController::Base include RequiresWhitelistedMonitoringClient - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true def index response = if Gitlab::Metrics.prometheus_metrics_enabled? diff --git a/app/controllers/omniauth_callbacks_controller.rb b/app/controllers/omniauth_callbacks_controller.rb index 27fd5f7ba37..ba62d2d5142 100644 --- a/app/controllers/omniauth_callbacks_controller.rb +++ b/app/controllers/omniauth_callbacks_controller.rb @@ -2,7 +2,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController include AuthenticatesWithTwoFactor include Devise::Controllers::Rememberable - protect_from_forgery except: [:kerberos, :saml, :cas3] + protect_from_forgery except: [:kerberos, :saml, :cas3], prepend: true def handle_omniauth omniauth_flow(Gitlab::Auth::OAuth) diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index 1a339f76d26..7aa277b3614 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -3,21 +3,27 @@ class SessionsController < Devise::SessionsController include AuthenticatesWithTwoFactor include Devise::Controllers::Rememberable include Recaptcha::ClientHelper + include Recaptcha::Verify skip_before_action :check_two_factor_requirement, only: [:destroy] prepend_before_action :check_initial_setup, only: [:new] prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create] + prepend_before_action :check_captcha, only: [:create] prepend_before_action :store_redirect_uri, only: [:new] + prepend_before_action :ldap_servers, only: [:new, :create] before_action :auto_sign_in_with_provider, only: [:new] before_action :load_recaptcha after_action :log_failed_login, only: [:new], if: :failed_login? + helper_method :captcha_enabled? + + CAPTCHA_HEADER = 'X-GitLab-Show-Login-Captcha'.freeze + def new set_minimum_password_length - @ldap_servers = Gitlab::Auth::LDAP::Config.available_servers super end @@ -46,6 +52,25 @@ class SessionsController < Devise::SessionsController private + def captcha_enabled? + request.headers[CAPTCHA_HEADER] && Gitlab::Recaptcha.enabled? + end + + # From https://github.com/plataformatec/devise/wiki/How-To:-Use-Recaptcha-with-Devise#devisepasswordscontroller + def check_captcha + return unless user_params[:password].present? + return unless captcha_enabled? + return unless Gitlab::Recaptcha.load_configurations! + + unless verify_recaptcha + self.resource = resource_class.new + flash[:alert] = 'There was an error with the reCAPTCHA. Please solve the reCAPTCHA again.' + flash.delete :recaptcha_error + + respond_with_navigational(resource) { render :new } + end + end + def log_failed_login Gitlab::AppLogger.info("Failed Login: username=#{user_params[:login]} ip=#{request.remote_ip}") end @@ -152,6 +177,10 @@ class SessionsController < Devise::SessionsController Gitlab::Recaptcha.load_configurations! end + def ldap_servers + @ldap_servers ||= Gitlab::Auth::LDAP::Config.available_servers + end + def authentication_method if user_params[:otp_attempt] "two-factor" diff --git a/app/views/devise/sessions/_new_base.html.haml b/app/views/devise/sessions/_new_base.html.haml index c45d2214592..0ee563ac066 100644 --- a/app/views/devise/sessions/_new_base.html.haml +++ b/app/views/devise/sessions/_new_base.html.haml @@ -12,5 +12,9 @@ %span Remember me .float-right.forgot-password = link_to "Forgot your password?", new_password_path(:user) + %div + - if captcha_enabled? + = recaptcha_tags + .submit-container.move-submit-down = f.submit "Sign in", class: "btn btn-save" diff --git a/changelogs/unreleased/bjk-48176_ruby_gc.yml b/changelogs/unreleased/bjk-48176_ruby_gc.yml new file mode 100644 index 00000000000..45c6338df81 --- /dev/null +++ b/changelogs/unreleased/bjk-48176_ruby_gc.yml @@ -0,0 +1,5 @@ +--- +title: Cleanup Prometheus ruby metrics +merge_request: 20039 +author: Ben Kochie +type: fixed diff --git a/changelogs/unreleased/blackst0ne-fix-protect-from-forgery-in-application-controller.yml b/changelogs/unreleased/blackst0ne-fix-protect-from-forgery-in-application-controller.yml new file mode 100644 index 00000000000..da75ea8b09e --- /dev/null +++ b/changelogs/unreleased/blackst0ne-fix-protect-from-forgery-in-application-controller.yml @@ -0,0 +1,5 @@ +--- +title: "[Rails5] Force the callback run first" +merge_request: 20055 +author: "@blackst0ne" +type: fixed diff --git a/changelogs/unreleased/blackst0ne-rails5-fix-data-store-spec.yml b/changelogs/unreleased/blackst0ne-rails5-fix-data-store-spec.yml new file mode 100644 index 00000000000..403c3764321 --- /dev/null +++ b/changelogs/unreleased/blackst0ne-rails5-fix-data-store-spec.yml @@ -0,0 +1,5 @@ +--- +title: '[Rails5] Fix "-1 is not a valid data_store"' +merge_request: 19917 +author: "@blackst0ne" +type: fixed diff --git a/changelogs/unreleased/fix-boards-issue-highlight.yml b/changelogs/unreleased/fix-boards-issue-highlight.yml new file mode 100644 index 00000000000..0cc3faa81ca --- /dev/null +++ b/changelogs/unreleased/fix-boards-issue-highlight.yml @@ -0,0 +1,5 @@ +--- +title: Fix boards issue highlight +merge_request: 20063 +author: George Tsiolis +type: changed diff --git a/changelogs/unreleased/no-multi-assign-enable.yml b/changelogs/unreleased/no-multi-assign-enable.yml new file mode 100644 index 00000000000..bb9c69b18e7 --- /dev/null +++ b/changelogs/unreleased/no-multi-assign-enable.yml @@ -0,0 +1,5 @@ +--- +title: Enable no-multi-assignment in JS files +merge_request: 19808 +author: gfyoung +type: other diff --git a/changelogs/unreleased/no-restricted-globals-enable.yml b/changelogs/unreleased/no-restricted-globals-enable.yml new file mode 100644 index 00000000000..1fa2eac0d03 --- /dev/null +++ b/changelogs/unreleased/no-restricted-globals-enable.yml @@ -0,0 +1,5 @@ +--- +title: Enable no-restricted globals in JS files +merge_request: 19877 +author: gfyoung +type: other diff --git a/config/karma.config.js b/config/karma.config.js index 28a688797d9..84810332dc2 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -15,6 +15,7 @@ function fatalError(message) { // disable problematic options webpackConfig.entry = undefined; webpackConfig.mode = 'development'; +webpackConfig.optimization.nodeEnv = false; webpackConfig.optimization.runtimeChunk = false; webpackConfig.optimization.splitChunks = false; diff --git a/config/webpack.config.js b/config/webpack.config.js index 79b0df76205..583f05f2fb7 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -174,7 +174,6 @@ module.exports = { }, optimization: { - nodeEnv: false, runtimeChunk: 'single', splitChunks: { maxInitialRequests: 4, diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 411a0fae93f..cea6764df41 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -49,6 +49,20 @@ The following metrics are available: | filesystem_circuitbreaker | Gauge | 9.5 | Whether or not the circuit for a certain shard is broken or not | | circuitbreaker_storage_check_duration_seconds | Histogram | 10.3 | Time a single storage probe took | +### Ruby metrics + +Some basic Ruby runtime metrics are available: + +| Metric | Type | Since | Description | +|:-------------------------------------- |:--------- |:----- |:----------- | +| ruby_gc_duration_seconds_total | Counter | 11.1 | Time spent by Ruby in GC | +| ruby_gc_stat_... | Gauge | 11.1 | Various metrics from [GC.stat] | +| ruby_file_descriptors | Gauge | 11.1 | File descriptors per process | +| ruby_memory_bytes | Gauge | 11.1 | Memory usage by process | +| ruby_sampler_duration_seconds_total | Counter | 11.1 | Time spent collecting stats | + +[GC.stat]: https://ruby-doc.org/core-2.3.0/GC.html#method-c-stat + ## Metrics shared directory GitLab's Prometheus client requires a directory to store metrics data shared between multi-process services. diff --git a/doc/api/runners.md b/doc/api/runners.md index 3ca07ce9795..ac814bbf19a 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -30,6 +30,7 @@ Example response: "description": "test-1-20150125", "id": 6, "is_shared": false, + "ip_address": "127.0.0.1", "name": null, "online": true, "status": "online" @@ -38,6 +39,7 @@ Example response: "active": true, "description": "test-2-20150125", "id": 8, + "ip_address": "127.0.0.1", "is_shared": false, "name": null, "online": false, @@ -72,6 +74,7 @@ Example response: "active": true, "description": "shared-runner-1", "id": 1, + "ip_address": "127.0.0.1", "is_shared": true, "name": null, "online": true, @@ -81,6 +84,7 @@ Example response: "active": true, "description": "shared-runner-2", "id": 3, + "ip_address": "127.0.0.1", "is_shared": true, "name": null, "online": false @@ -90,6 +94,7 @@ Example response: "active": true, "description": "test-1-20150125", "id": 6, + "ip_address": "127.0.0.1", "is_shared": false, "name": null, "online": true @@ -99,6 +104,7 @@ Example response: "active": true, "description": "test-2-20150125", "id": 8, + "ip_address": "127.0.0.1", "is_shared": false, "name": null, "online": false, @@ -131,6 +137,7 @@ Example response: "architecture": null, "description": "test-1-20150125", "id": 6, + "ip_address": "127.0.0.1", "is_shared": false, "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, @@ -189,6 +196,7 @@ Example response: "architecture": null, "description": "test-1-20150125-test", "id": 6, + "ip_address": "127.0.0.1", "is_shared": false, "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, @@ -257,6 +265,7 @@ Example response: [ { "id": 2, + "ip_address": "127.0.0.1", "status": "running", "stage": "test", "name": "test", @@ -345,6 +354,7 @@ Example response: "active": true, "description": "test-2-20150125", "id": 8, + "ip_address": "127.0.0.1", "is_shared": false, "name": null, "online": false, @@ -354,6 +364,7 @@ Example response: "active": true, "description": "development_runner", "id": 5, + "ip_address": "127.0.0.1", "is_shared": true, "name": null, "online": true @@ -386,6 +397,7 @@ Example response: "active": true, "description": "test-2016-02-01", "id": 9, + "ip_address": "127.0.0.1", "is_shared": false, "name": null, "online": true, diff --git a/doc/development/utilities.md b/doc/development/utilities.md index 8f9aff1a35f..0d074a3ef05 100644 --- a/doc/development/utilities.md +++ b/doc/development/utilities.md @@ -135,3 +135,44 @@ We developed a number of utilities to ease development. Find.new.clear_memoization(:result) ``` + +## [`RequestCache`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/cache/request_cache.rb) + +This module provides a simple way to cache values in RequestStore, +and the cache key would be based on the class name, method name, +optionally customized instance level values, optionally customized +method level values, and optional method arguments. + +A simple example that only uses the instance level customised values: + +``` ruby +class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + + request_cache def can_push_to_branch?(ref) + # ... + end +end +``` + +This way, the result of `can_push_to_branch?` would be cached in +`RequestStore.store` based on the cache key. If `RequestStore` is not +currently active, then it would be stored in a hash saved in an +instance variable, so the cache logic would be the same. + +We can also set different strategies for different methods: + +``` ruby +class Commit + extend Gitlab::Cache::RequestCache + + def author + User.find_by_any_email(author_email.downcase) + end + request_cache(:author) { author_email.downcase } +end +``` diff --git a/doc/integration/recaptcha.md b/doc/integration/recaptcha.md index a301d1a613c..932cd479d56 100644 --- a/doc/integration/recaptcha.md +++ b/doc/integration/recaptcha.md @@ -20,4 +20,21 @@ To use reCAPTCHA, first you must create a site and private key. 6. Check the `Enable reCAPTCHA` checkbox -7. Save the configuration. +7. Save the configuration. + +## Enabling reCAPTCHA for user logins via passwords + +By default, reCAPTCHA is only enabled for user registrations. To enable it for +user logins via passwords, the `X-GitLab-Show-Login-Captcha` HTTP header must +be set. For example, in NGINX, this can be done via the `proxy_set_header` +configuration variable: + +``` +proxy_set_header X-GitLab-Show-Login-Captcha 1; +``` + +In GitLab Omnibus, this can be configured via `/etc/gitlab/gitlab.rb`: + +```ruby +nginx['proxy_set_headers'] = { 'X-GitLab-Show-Login-Captcha' => 1 } +``` diff --git a/doc/topics/autodevops/img/auto_monitoring.png b/doc/topics/autodevops/img/auto_monitoring.png Binary files differindex 92902e3ca72..2900e5d1877 100644 --- a/doc/topics/autodevops/img/auto_monitoring.png +++ b/doc/topics/autodevops/img/auto_monitoring.png diff --git a/doc/topics/autodevops/img/guide_choose_gke.png b/doc/topics/autodevops/img/guide_choose_gke.png Binary files differnew file mode 100644 index 00000000000..6da3a7220da --- /dev/null +++ b/doc/topics/autodevops/img/guide_choose_gke.png diff --git a/doc/topics/autodevops/img/guide_cluster_apps.png b/doc/topics/autodevops/img/guide_cluster_apps.png Binary files differnew file mode 100644 index 00000000000..33d25f2950d --- /dev/null +++ b/doc/topics/autodevops/img/guide_cluster_apps.png diff --git a/doc/topics/autodevops/img/guide_connect_cluster.png b/doc/topics/autodevops/img/guide_connect_cluster.png Binary files differindex b856b81a1d0..703d536f37a 100644 --- a/doc/topics/autodevops/img/guide_connect_cluster.png +++ b/doc/topics/autodevops/img/guide_connect_cluster.png diff --git a/doc/topics/autodevops/img/guide_create_cluster.png b/doc/topics/autodevops/img/guide_create_cluster.png Binary files differnew file mode 100644 index 00000000000..cd1d0fdd8da --- /dev/null +++ b/doc/topics/autodevops/img/guide_create_cluster.png diff --git a/doc/topics/autodevops/img/guide_create_project.png b/doc/topics/autodevops/img/guide_create_project.png Binary files differnew file mode 100644 index 00000000000..4ed1071db03 --- /dev/null +++ b/doc/topics/autodevops/img/guide_create_project.png diff --git a/doc/topics/autodevops/img/guide_enable_autodevops.png b/doc/topics/autodevops/img/guide_enable_autodevops.png Binary files differnew file mode 100644 index 00000000000..0fc3ecca19a --- /dev/null +++ b/doc/topics/autodevops/img/guide_enable_autodevops.png diff --git a/doc/topics/autodevops/img/guide_environments.png b/doc/topics/autodevops/img/guide_environments.png Binary files differnew file mode 100644 index 00000000000..1d8d5614e64 --- /dev/null +++ b/doc/topics/autodevops/img/guide_environments.png diff --git a/doc/topics/autodevops/img/guide_environments_metrics.png b/doc/topics/autodevops/img/guide_environments_metrics.png Binary files differnew file mode 100644 index 00000000000..f0d31f31581 --- /dev/null +++ b/doc/topics/autodevops/img/guide_environments_metrics.png diff --git a/doc/topics/autodevops/img/guide_first_pipeline.png b/doc/topics/autodevops/img/guide_first_pipeline.png Binary files differnew file mode 100644 index 00000000000..57459dcc9d9 --- /dev/null +++ b/doc/topics/autodevops/img/guide_first_pipeline.png diff --git a/doc/topics/autodevops/img/guide_gitlab_gke_details.png b/doc/topics/autodevops/img/guide_gitlab_gke_details.png Binary files differnew file mode 100644 index 00000000000..bc5a53800f7 --- /dev/null +++ b/doc/topics/autodevops/img/guide_gitlab_gke_details.png diff --git a/doc/topics/autodevops/img/guide_gke_apis_after.png b/doc/topics/autodevops/img/guide_gke_apis_after.png Binary files differnew file mode 100644 index 00000000000..380de958867 --- /dev/null +++ b/doc/topics/autodevops/img/guide_gke_apis_after.png diff --git a/doc/topics/autodevops/img/guide_gke_apis_before.png b/doc/topics/autodevops/img/guide_gke_apis_before.png Binary files differnew file mode 100644 index 00000000000..d06fc707887 --- /dev/null +++ b/doc/topics/autodevops/img/guide_gke_apis_before.png diff --git a/doc/topics/autodevops/img/guide_google_auth.png b/doc/topics/autodevops/img/guide_google_auth.png Binary files differnew file mode 100644 index 00000000000..b97b2be9f15 --- /dev/null +++ b/doc/topics/autodevops/img/guide_google_auth.png diff --git a/doc/topics/autodevops/img/guide_google_signin.png b/doc/topics/autodevops/img/guide_google_signin.png Binary files differnew file mode 100644 index 00000000000..e59fc94bd4c --- /dev/null +++ b/doc/topics/autodevops/img/guide_google_signin.png diff --git a/doc/topics/autodevops/img/guide_ide_commit.png b/doc/topics/autodevops/img/guide_ide_commit.png Binary files differnew file mode 100644 index 00000000000..188f60f2a4b --- /dev/null +++ b/doc/topics/autodevops/img/guide_ide_commit.png diff --git a/doc/topics/autodevops/img/guide_integration.png b/doc/topics/autodevops/img/guide_integration.png Binary files differdeleted file mode 100644 index 723b2619ea2..00000000000 --- a/doc/topics/autodevops/img/guide_integration.png +++ /dev/null diff --git a/doc/topics/autodevops/img/guide_merge_request.png b/doc/topics/autodevops/img/guide_merge_request.png Binary files differnew file mode 100644 index 00000000000..d78e69be776 --- /dev/null +++ b/doc/topics/autodevops/img/guide_merge_request.png diff --git a/doc/topics/autodevops/img/guide_merge_request_ide.png b/doc/topics/autodevops/img/guide_merge_request_ide.png Binary files differnew file mode 100644 index 00000000000..c825b0849e1 --- /dev/null +++ b/doc/topics/autodevops/img/guide_merge_request_ide.png diff --git a/doc/topics/autodevops/img/guide_merge_request_review_app.png b/doc/topics/autodevops/img/guide_merge_request_review_app.png Binary files differnew file mode 100644 index 00000000000..1b9b854ddac --- /dev/null +++ b/doc/topics/autodevops/img/guide_merge_request_review_app.png diff --git a/doc/topics/autodevops/img/guide_pipeline_stages.png b/doc/topics/autodevops/img/guide_pipeline_stages.png Binary files differnew file mode 100644 index 00000000000..6e2f078152b --- /dev/null +++ b/doc/topics/autodevops/img/guide_pipeline_stages.png diff --git a/doc/topics/autodevops/img/guide_project_landing_page.png b/doc/topics/autodevops/img/guide_project_landing_page.png Binary files differnew file mode 100644 index 00000000000..4f8d2eb10b1 --- /dev/null +++ b/doc/topics/autodevops/img/guide_project_landing_page.png diff --git a/doc/topics/autodevops/img/guide_project_template.png b/doc/topics/autodevops/img/guide_project_template.png Binary files differnew file mode 100644 index 00000000000..298ac0f6fcf --- /dev/null +++ b/doc/topics/autodevops/img/guide_project_template.png diff --git a/doc/topics/autodevops/img/guide_secret.png b/doc/topics/autodevops/img/guide_secret.png Binary files differdeleted file mode 100644 index 01f5aa49908..00000000000 --- a/doc/topics/autodevops/img/guide_secret.png +++ /dev/null diff --git a/doc/topics/autodevops/img/rollout_staging_disabled.png b/doc/topics/autodevops/img/rollout_staging_disabled.png Binary files differindex 71e36b440f0..4c7c6768666 100644 --- a/doc/topics/autodevops/img/rollout_staging_disabled.png +++ b/doc/topics/autodevops/img/rollout_staging_disabled.png diff --git a/doc/topics/autodevops/img/rollout_staging_enabled.png b/doc/topics/autodevops/img/rollout_staging_enabled.png Binary files differindex d0d1d356627..f45c1c2cb37 100644 --- a/doc/topics/autodevops/img/rollout_staging_enabled.png +++ b/doc/topics/autodevops/img/rollout_staging_enabled.png diff --git a/doc/topics/autodevops/img/staging_enabled.png b/doc/topics/autodevops/img/staging_enabled.png Binary files differindex 0ef1a67d641..f0e0cd1cfcd 100644 --- a/doc/topics/autodevops/img/staging_enabled.png +++ b/doc/topics/autodevops/img/staging_enabled.png diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 103836e59d0..1d26a743500 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -1,6 +1,6 @@ # Auto DevOps -> [Introduced][ce-37115] in GitLab 10.0. +> [Introduced][ce-37115] in GitLab 10.0. Generally available on GitLab 11.0. Auto DevOps automatically detects, builds, tests, deploys, and monitors your applications. @@ -13,6 +13,12 @@ without needing to configure anything. Just push your code and GitLab takes care of everything else. This makes it easier to start new projects and brings consistency to how applications are set up throughout a company. +## Quick start + +If you are using GitLab.com, see the [quick start guide](quick_start_guide.md) +for using Auto DevOps with GitLab.com and a Kubernetes cluster on Google Kubernetes +Engine. + ## Comparison to application platforms and PaaS Auto DevOps provides functionality described by others as an application @@ -34,19 +40,19 @@ in a couple of ways: ## Features Comprised of a set of stages, Auto DevOps brings these best practices to your -project in an easy and automatic way: +project in a simple and automatic way: 1. [Auto Build](#auto-build) 1. [Auto Test](#auto-test) -1. [Auto Code Quality](#auto-code-quality) -1. [Auto SAST (Static Application Security Testing)](#auto-sast) -1. [Auto Dependency Scanning](#auto-dependency-scanning) -1. [Auto License Management](#auto-license-management) +1. [Auto Code Quality](#auto-code-quality) **[STARTER]** +1. [Auto SAST (Static Application Security Testing)](#auto-sast) **[ULTIMATE]** +1. [Auto Dependency Scanning](#auto-dependency-scanning) **[ULTIMATE]** +1. [Auto License Management](#auto-license-management) **[ULTIMATE]** 1. [Auto Container Scanning](#auto-container-scanning) 1. [Auto Review Apps](#auto-review-apps) -1. [Auto DAST (Dynamic Application Security Testing)](#auto-dast) +1. [Auto DAST (Dynamic Application Security Testing)](#auto-dast) **[ULTIMATE]** 1. [Auto Deploy](#auto-deploy) -1. [Auto Browser Performance Testing](#auto-browser-performance-testing) +1. [Auto Browser Performance Testing](#auto-browser-performance-testing) **[PREMIUM]** 1. [Auto Monitoring](#auto-monitoring) As Auto DevOps relies on many different components, it's good to have a basic @@ -135,10 +141,9 @@ and `1.2.3.4` is the IP address of your load balancer; generally NGINX ([see requirements](#requirements)). How to set up the DNS record is beyond the scope of this document; you should check with your DNS provider. -Alternatively you can use free public services like [nip.io](http://nip.io) or -[nip.io](http://nip.io) which provide automatic wildcard DNS without any -configuration. Just set the Auto DevOps base domain to `1.2.3.4.nip.io` or -`1.2.3.4.nip.io`. +Alternatively you can use free public services like [nip.io](http://nip.io) +which provide automatic wildcard DNS without any configuration. Just set the +Auto DevOps base domain to `1.2.3.4.nip.io`. Once set up, all requests will hit the load balancer, which in turn will route them to the Kubernetes pods that run your application(s). @@ -198,12 +203,6 @@ and verifying that your app is deployed as a review app in the Kubernetes cluster with the `review/*` environment scope. Similarly, you can check the other environments. -## Quick start - -If you are using GitLab.com, see our [quick start guide](quick_start_guide.md) -for using Auto DevOps with GitLab.com and an external Kubernetes cluster on -Google Cloud. - ## Enabling Auto DevOps If you haven't done already, read the [requirements](#requirements) to make @@ -288,7 +287,7 @@ NOTE: **Note:** Auto Test uses tests you already have in your application. If there are no tests, it's up to you to add them. -### Auto Code Quality +### Auto Code Quality **[STARTER]** Auto Code Quality uses the [Code Quality image](https://gitlab.com/gitlab-org/security-products/codequality) to run @@ -323,7 +322,7 @@ to run analysis on the project dependencies and checks for potential security is report is created, it's uploaded as an artifact which you can later download and check out. -In GitLab Ultimate, any security warnings are also +Any security warnings are also [shown in the merge request widget](https://docs.gitlab.com/ee//user/project/merge_requests/dependency_scanning.html). ### Auto License Management **[ULTIMATE]** @@ -331,12 +330,12 @@ In GitLab Ultimate, any security warnings are also > Introduced in [GitLab Ultimate][ee] 11.0. License Management uses the -[License Management Docker image](https://gitlab.com/gitlab-org/security-products/license_management) +[License Management Docker image](https://gitlab.com/gitlab-org/security-products/license-management) to search the project dependencies for their license. Once the report is created, it's uploaded as an artifact which you can later download and check out. -In GitLab Ultimate, any licenses are also +Any licenses are also [shown in the merge request widget](https://docs.gitlab.com/ee//user/project/merge_requests/license_management.html). ### Auto Container Scanning diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 61c04f3d9bb..44b0cf758dc 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -1,143 +1,290 @@ -# Auto DevOps: quick start guide +# Getting started with Auto DevOps -> [Introduced][ce-37115] in GitLab 10.0. +This is a step-by-step guide that will help you use [Auto DevOps](index.md) to +deploy a project hosted on GitLab.com to Google Kubernetes Engine. -This is a step-by-step guide to deploying a project hosted on GitLab.com to -Google Cloud, using Auto DevOps. +We will use GitLab's native Kubernetes integration, so you will not need +to create a Kubernetes cluster manually using the Google Cloud Platform console. +We will create and deploy a simple application that we create from a GitLab template. -We made a minimal [Ruby -application](https://gitlab.com/auto-devops-examples/minimal-ruby-app) to use -as an example for this guide. It contains two main files: +These instructions will also work for a self-hosted GitLab instance; you'll just +need to ensure your own [Runners are configured](../../ci/runners/README.md) and +[Google OAuth is enabled](../../integration/google.md). -* `server.rb` - our application. It will start an HTTP server on port 5000 and - render "Hello, world!" -* `Dockerfile` - to build our app into a container image. It will use a ruby - base image and run `server.rb` +## Configuring your Google account -## Fork sample project on GitLab.com +Before creating and connecting your Kubernetes cluster to your GitLab project, +you need a Google Cloud Platform account. If you don't already have one, +sign up at https://console.cloud.google.com. You'll need to either sign in with an existing +Google account (for example, one that you use to access Gmail, Drive, etc.) or create a new one. -Let’s start by forking our sample application. Go to [the project -page](https://gitlab.com/auto-devops-examples/minimal-ruby-app) and press the -**Fork** button. Soon you should have a project under your namespace with the -necessary files. +1. Follow the steps as outlined in the ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) + in order for the required APIs and related services to be enabled. +1. Make sure you have created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account). -You can also start a new project from a -[GitLab project template](https://gitlab.com/gitlab-org/project-templates) if -you want to use a different language. +TIP: **Tip:** +Every new Google Cloud Platform (GCP) account receives [$300 in credit](https://console.cloud.google.com/freetrial), +and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's +Google Kubernetes Engine Integration. All you have to do is [follow this link](https://goo.gl/AaJzRW) and apply for credit. -## Setup your own cluster on Google Kubernetes Engine +## Creating a new project from a template -If you do not already have a Google Cloud account, create one at -https://console.cloud.google.com. +We will use one of GitLab's project templates to get started. As the name suggests, +those projects provide a barebones application built on some well-known frameworks. -Visit the [**Kubernetes Engine**](https://console.cloud.google.com/kubernetes/list) -tab and create a new cluster. You can change the name and leave the rest of the -default settings. Once you have your cluster running, you need to connect to the -cluster by following the Google interface. +1. In GitLab, click the plus icon (**+**) at the top of the navigation bar and select + **New project**. +1. Go to the **Create from template** tab where you can choose among a Ruby on + Rails, Spring, or NodeJS Express project. For this example, + we'll use the Ruby on Rails template. -## Connect to Kubernetes cluster +  -You need to have the Google Cloud SDK installed. e.g. -On macOS, install [homebrew](https://brew.sh): +1. Give your project a name, optionally a description, and make it public so that + you can take advantage of the features available in the + [GitLab Gold plan](https://about.gitlab.com/pricing/#gitlab-com). -1. Install Brew Caskroom: `brew install caskroom/cask/brew-cask` -2. Install Google Cloud SDK: `brew cask install google-cloud-sdk` -3. Add `kubectl` with: `gcloud components install kubectl` -4. Log in: `gcloud auth login` +  -Now go back to the Google interface, find your cluster, follow the instructions -under "Connect to the cluster" and open the Kubernetes Dashboard. It will look -something like: +1. Click **Create project**. -```sh -gcloud container clusters get-credentials ruby-autodeploy \ --zone europe-west2-c --project api-project-XXXXXXX -``` +Now that the project is created, the next step is to create the Kubernetes cluster +under which this application will be deployed. -Finally, run `kubectl proxy`. +## Creating a Kubernetes cluster from within GitLab - +1. On the project's landing page, click the button labeled **Add Kubernetes cluster** + (note that this option is also available when you navigate to **Operations > Kubernetes**). -## Copy credentials to GitLab.com project +  -Once you have the Kubernetes Dashboard interface running, you should visit -**Secrets** under the "Config" section. There, you should find the settings we -need for GitLab integration: `ca.crt` and token. +1. Choose **Create on Google Kubernetes Engine**. - +  -You need to copy-paste the `ca.crt` and token into your project on GitLab.com in -the Kubernetes integration page under project -**Settings > Integrations > Project services > Kubernetes**. Don't actually copy -the namespace though. Each project should have a unique namespace, and by leaving -it blank, GitLab will create one for you. +1. Sign in with Google. - +  -For the API URL, you should use the "Endpoint" IP from your cluster page on -Google Cloud Platform. +1. Connect with your Google account and press **Allow** when asked (this will + be shown only the first time you connect GitLab with your Google account). -## Expose application to the world +  -In order to be able to visit your application, you need to install an NGINX -ingress controller and point your domain name to its external IP address. Let's -see how that's done. +1. The last step is to fill in the cluster details. Give it a name, leave the + environment scope as is, and choose the GCP project under which the cluster + will be created. (Per the instructions when you + [configured your Google account](#configuring-your-google-account), a project + should have already been created for you.) Next, choose the + [region/zone](https://cloud.google.com/compute/docs/regions-zones/) under which the + cluster will be created, enter the number of nodes you want it to have, and + finally choose their [machine type](https://cloud.google.com/compute/docs/machine-types). -### Set up Ingress controller +  -You’ll need to make sure you have an ingress controller. If you don’t have one, do: +1. Once ready, click **Create Kubernetes cluster**. -```sh -brew install kubernetes-helm -helm init -helm install --name ruby-app stable/nginx-ingress -``` +After a couple of minutes, the cluster will be created. You can also see its +status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). -This should create several services including `ruby-app-nginx-ingress-controller`. -You can list your services by running `kubectl get svc` to confirm that. +The next step is to install some applications on your cluster that are needed +to take full advantage of Auto DevOps. -### Point DNS at Cluster IP +## Installing Helm, Ingress, and Prometheus -Find out the external IP address of the `ruby-app-nginx-ingress-controller` by -running: +GitLab's Kubernetes integration comes with some +[pre-defined applications](../../user/project/clusters/index.md#installing-applications) +for you to install. -```sh -kubectl get svc ruby-app-nginx-ingress-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}' -``` + + +The first one to install is Helm Tiller, a package manager for Kubernetes, which +is needed in order to install the rest of the applications. Go ahead and click +its **Install** button. + +Once it's installed, the other applications that rely on it will each have their **Install** +button enabled. For this guide, we need Ingress and Prometheus. Ingress provides +load balancing, SSL termination, and name-based virtual hosting, using NGINX behind +the scenes. Prometheus is an open-source monitoring and alerting system that we'll +use to supervise the deployed application. We will not install GitLab Runner as +we'll use the shared Runners that GitLab.com provides. + +After the Ingress is installed, wait a few seconds and copy the IP address that +is displayed, which we'll use in the next step when enabling Auto DevOps. + +## Enabling Auto DevOps + +Now that the Kubernetes cluster is set up and ready, let's enable Auto DevOps. + +1. First, navigate to **Settings > CI/CD > Auto DevOps**. +1. Select **Enable Auto DevOps**. +1. Add in your base **Domain** by using the one GitLab suggests. Note that + generally, you would associate the IP address with a domain name on your + registrar's settings. In this case, for the sake of the guide, we will use + an alternative DNS that will map any domain name of the scheme + `anything.ip_address.nip.io` to the corresponding `ip_address`. For example, + if the IP address of the Ingress is `1.2.3.4`, the domain name to fill in + would be `1.2.3.4.nip.io`. +1. Lastly, let's select the [continuous deployment strategy](index.md#deployment-strategy) + which will automatically deploy the application to production once the pipeline + successfully runs on the `master` branch. +1. Click **Save changes**. + +  + +Once you complete all the above and save your changes, a new pipeline is +automatically created. To view the pipeline, go to **CI/CD > Pipelines**. + + + +In the next section we'll break down the pipeline and explain what each job does. + +## Deploying the application + +By now you should see the pipeline running, but what is it running exactly? + +To navigate inside the pipeline, click its status badge. (It's status should be "running"). +The pipeline is split into 4 stages, each running a couple of jobs. + + + +In the **build** stage, the application is built into a Docker image and then +uploaded to your project's [Container Registry](../../user/project/container_registry.md) ([Auto Build](index.md#auto-build)). + +In the **test** stage, GitLab runs various checks on the application: + +- The `test` job runs unit and integration tests by detecting the language and + framework ([Auto Test](index.md#auto-test)) +- The `code_quality` job checks the code quality and is allowed to fail + ([Auto Code Quality](index.md#auto-code-quality)) **[STARTER]** +- The `container_scanning` job checks the Docker container if it has any + vulnerabilities and is allowed to fail ([Auto Container Scanning](index.md#auto-container-scanning)) +- The `dependency_scanning` job checks if the application has any dependencies + susceptible to vulnerabilities and is allowed to fail ([Auto Dependency Scanning](index.md#auto-dependency-scanning)) **[ULTIMATE]** +- The `sast` job runs static analysis on the current code to check for potential + security issues and is allowed to fail([Auto SAST](index.md#auto-sast)) **[ULTIMATE]** +- The `license_management` job searches the application's dependencies to determine each of their + licenses and is allowed to fail ([Auto License Management](index.md#auto-license-management)) **[ULTIMATE]** NOTE: **Note:** -If your ingress controller has been installed in a different way, you can find -how to get the external IP address in the -[Cluster documentation](../../user/project/clusters/index.md#getting-the-external-ip-address). +As you might have noticed, all jobs except `test` are allowed to fail in the +test stage. + +The **production** stage is run after the tests and checks finish, and it automatically +deploys the application in Kubernetes ([Auto Deploy](index.md#auto-deploy)). + +Lastly, in the **performance** stage, some performance tests will run +on the deployed application +([Auto Browser Performance Testing](index.md#auto-browser-performance-testing)). **[PREMIUM]** + +--- -Use this IP address to configure your DNS. This part heavily depends on your -preferences and domain provider. But in case you are not sure, just create an -A record with a wildcard host like `*.<your-domain>`. +The URL for the deployed application can be found under the **Environments** +page where you can also monitor your application. Let's explore that. + +### Monitoring + +Now that the application is successfully deployed, let's navigate to its +website. First, go to **Operations > Environments**. + + + +In **Environments** you can see some details about the deployed +applications. In the rightmost column for the production environment, you can make use of the three icons: + +- The first icon will open the URL of the application that is deployed in + production. It's a very simple page, but the important part is that it works! +- The next icon with the small graph will take you to the metrics page where + Prometheus collects data about the Kubernetes cluster and how the application + affects it (in terms of memory/CPU usage, latency, etc.). + +  + +- The third icon is the [web terminal](../../ci/environments.md#web-terminals) + and it will open a terminal session right inside the container where the + application is running. + +Right below, there is the +[Deploy Board](https://docs.gitlab.com/ee/user/project/deploy_boards.md). +The squares represent pods in your Kubernetes cluster that are associated with +the given environment. Hovering above each square you can see the state of a +deployment and clicking a square will take you to the pod's logs page. + +TIP: **Tip:** +There is only one pod hosting the application at the moment, but you can add +more pods by defining the [`REPLICAS` variable](index.md#environment-variables) +under **Settings > CI/CD > Variables**. + +### Working with branches + +Following the [GitLab flow](../../workflow/gitlab_flow.md#working-with-feature-branches) +let's create a feature branch that will add some content to the application. + +Under your repository, navigate to the following file: `app/views/welcome/index.html.erb`. +By now, it should only contain a paragraph: `<p>You're on Rails!</p>`, so let's +start adding content. Let's use GitLab's [Web IDE](../../user/project/web_ide/index.md) to make the change. Once +you're on the Web IDE, make the following change: + +```html +<p>You're on Rails! Powered by GitLab Auto DevOps.</p> +``` + +Stage the file, add a commit message, and create a new branch and a merge request +by clicking **Commit**. + + + +Once you submit the merge request, you'll see the pipeline running. This will +run all the jobs as [described previously](#deploying-the-application), as well +a few more that run only on branches other than `master`. + + + +After a few minutes you'll notice that there was a failure in a test. +This means there's a test that was 'broken' by our change. +Navigating into the `test` job that failed, you can see what the broken test is: + +``` +Failure: +WelcomeControllerTest#test_should_get_index [/app/test/controllers/welcome_controller_test.rb:7]: +<You're on Rails!> expected but was +<You're on Rails! Powered by GitLab Auto DevOps.>.. +Expected 0 to be >= 1. + +bin/rails test test/controllers/welcome_controller_test.rb:4 +``` -Use `nslookup minimal-ruby-app-staging.<yourdomain>` to confirm that domain is -assigned to the cluster IP. +Let's fix that: -## Set up Auto DevOps +1. Back to the merge request, click the **Web IDE** button. +1. Find the `test/controllers/welcome_controller_test.rb` file and open it. +1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` +1. Click **Commit**. +1. On your left, under "Unstaged changes", click the little checkmark icon + to stage the changes. +1. Write a commit message and click **Commit**. -In your GitLab.com project, go to **Settings > CI/CD** and find the Auto DevOps -section. Select "Enable Auto DevOps", add in your base domain, and save. +Now, if you go back to the merge request you should not only see the test passing, +but also the application deployed as a [review app](index.md#auto-review-apps). You +can visit it by following the URL in the merge request. The changes that we +previously made should be there. -Next, a pipeline needs to be triggered. Since the test project doesn't have a -`.gitlab-ci.yml`, you need to either push a change to the repository or -manually visit `https://gitlab.com/<username>/minimal-ruby-app/pipelines/new`, -where `<username>` is your username. + -This will create a new pipeline with several jobs: `build`, `test`, `code_quality`, -and `production`. The `build` job will create a Docker image with your new -change and push it to the Container Registry. The `test` job will test your -changes, whereas the `code_quality` job will run static analysis on your changes. -Finally, the `production` job will deploy your changes to a production application. +Once you merge the merge request, the pipeline will run on the `master` branch, +and the application will be eventually deployed straight to production. -Once the deploy job succeeds you should be able to see your application by -visiting the Kubernetes dashboard. Select the namespace of your project, which -will look like `minimal-ruby-app-23`, but with a unique ID for your project, -and your app will be listed as "production" under the Deployment tab. +## Conclusion -Once its ready, just visit `http://minimal-ruby-app.example.com` to see the -famous "Hello, world!"! +After implementing this project, you should now have a solid understanding of the basics of Auto DevOps. +We started from building and testing to deploying and monitoring an application +all within GitLab. Despite its automatic nature, Audo DevOps can also be configured +and customized to fit your workflow. Here are some helpful resources for further reading: -[ce-37115]: https://gitlab.com/gitlab-org/gitlab-ce/issues/37115 +1. [Auto DevOps](index.md) +1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters) **[PREMIUM]** +1. [Incremental rollout to production](index.md#incremental-rollout-to-production) **[PREMIUM]** +1. [Disable jobs you don't need with environment variables](index.md#environment-variables) +1. [Use a static IP for your cluster](../../user/project/clusters/index.md#using-a-static-ip) +1. [Use your own buildpacks to build your application](index.md#custom-buildpacks) +1. [Prometheus monitoring](../../user/project/integrations/prometheus.md) diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 58a483bb3b2..20c46cafbe5 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -7,9 +7,10 @@ cluster in a few steps. ## Overview -With a Kubernetes cluster associated to your project, you can use +With one or more Kubernetes clusters associated to your project, you can use [Review Apps](../../../ci/review_apps/index.md), deploy your applications, run -your pipelines, and much more, in an easy way. +your pipelines, use it with [Auto DevOps](../../../topics/autodevops/index.md), +and much more, all from within GitLab. There are two options when adding a new cluster to your project; either associate your account with Google Kubernetes Engine (GKE) so that you can [create new @@ -18,59 +19,65 @@ or provide the credentials to an [existing Kubernetes cluster](#adding-an-existi ## Adding and creating a new GKE cluster via GitLab +TIP: **Tip:** +Every new Google Cloud Platform (GCP) account receives [$300 in credit upon sign up](https://console.cloud.google.com/freetrial), +and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's +Google Kubernetes Engine Integration. All you have to do is [follow this link](https://goo.gl/AaJzRW) and apply for credit. + NOTE: **Note:** -You need Maintainer [permissions] and above to access the Kubernetes page. - -Before proceeding, make sure the following requirements are met: - -- The [Google authentication integration](../../../integration/google.md) must - be enabled in GitLab at the instance level. If that's not the case, ask your - GitLab administrator to enable it. -- Your associated Google account must have the right privileges to manage - clusters on GKE. That would mean that a [billing - account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) - must be set up and that you have to have permissions to access it. -- You must have Maintainer [permissions] in order to be able to access the - **Kubernetes** page. -- You must have [Cloud Billing API](https://cloud.google.com/billing/) enabled -- You must have [Resource Manager - API](https://cloud.google.com/resource-manager/) +The [Google authentication integration](../../../integration/google.md) must +be enabled in GitLab at the instance level. If that's not the case, ask your +GitLab administrator to enable it. On GitLab.com, this is enabled. + +### Requirements + +Before creating your first cluster on Google Kubernetes Engine with GitLab's +integration, make sure the following requirements are met: + +- A [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) + is set up and you have permissions to access it. +- The Kubernetes Engine API is enabled. Follow the steps as outlined in the + ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin). + +### Creating the cluster If all of the above requirements are met, you can proceed to create and add a -new Kubernetes cluster that will be hosted on GKE to your project: +new Kubernetes cluster to your project: 1. Navigate to your project's **Operations > Kubernetes** page. + + NOTE: **Note:** + You need Maintainer [permissions] and above to access the Kubernetes page. + 1. Click on **Add Kubernetes cluster**. 1. Click on **Create with Google Kubernetes Engine**. 1. Connect your Google account if you haven't done already by clicking the **Sign in with Google** button. -1. Fill in the requested values: +1. From there on, choose your cluster's settings: - **Kubernetes cluster name** - The name you wish to give the cluster. - **Environment scope** - The [associated environment](#setting-the-environment-scope) to this cluster. - - **Google Cloud Platform project** - The project you created in your GCP - console that will host the Kubernetes cluster. This must **not** be confused - with the project ID. Learn more about [Google Cloud Platform projects](https://cloud.google.com/resource-manager/docs/creating-managing-projects). - - **Zone** - The [zone](https://cloud.google.com/compute/docs/regions-zones/) + - **Google Cloud Platform project** - Choose the project you created in your GCP + console that will host the Kubernetes cluster. Learn more about + [Google Cloud Platform projects](https://cloud.google.com/resource-manager/docs/creating-managing-projects). + - **Zone** - Choose the [region zone](https://cloud.google.com/compute/docs/regions-zones/) under which the cluster will be created. - - **Number of nodes** - The number of nodes you wish the cluster to have. + - **Number of nodes** - Enter the number of nodes you wish the cluster to have. - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-types) of the Virtual Machine instance that the cluster will be based on. 1. Finally, click the **Create Kubernetes cluster** button. -After a few moments, your cluster should be created. If something goes wrong, -you will be notified. - -You can now proceed to install some pre-defined applications and then -enable the Cluster integration. +After a couple of minutes, your cluster will be ready to go. You can now proceed +to install some [pre-defined applications](#installing-applications). ## Adding an existing Kubernetes cluster -NOTE: **Note:** -You need Maintainer [permissions] and above to access the Kubernetes page. - To add an existing Kubernetes cluster to your project: 1. Navigate to your project's **Operations > Kubernetes** page. + + NOTE: **Note:** + You need Maintainer [permissions] and above to access the Kubernetes page. + 1. Click on **Add Kubernetes cluster**. 1. Click on **Add an existing Kubernetes cluster** and fill in the details: - **Kubernetes cluster name** (required) - The name you wish to give the cluster. @@ -91,9 +98,8 @@ To add an existing Kubernetes cluster to your project: to create one. You can also view or create service tokens in the [Kubernetes dashboard](https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#config) (under **Config > Secrets**). - - **Project namespace** (optional) - The following apply: - - By default you don't have to fill it in; by leaving it blank, GitLab will - create one for you. + - **Project namespace** (optional) - You don't have to fill it in; by leaving + it blank, GitLab will create one for you. Also: - Each project should have a unique namespace. - The project namespace is not necessarily the namespace of the secret, if you're using a secret with broader permissions, like the secret from `default`. @@ -103,11 +109,8 @@ To add an existing Kubernetes cluster to your project: be the same. 1. Finally, click the **Create Kubernetes cluster** button. -After a few moments, your cluster should be created. If something goes wrong, -you will be notified. - -You can now proceed to install some pre-defined applications and then -enable the Kubernetes cluster integration. +After a couple of minutes, your cluster will be ready to go. You can now proceed +to install some [pre-defined applications](#installing-applications). ## Security implications @@ -152,9 +155,9 @@ added directly to your configured cluster. Those applications are needed for | Application | GitLab version | Description | | ----------- | :------------: | ----------- | -| [Helm Tiller](https://docs.helm.sh/) | 10.2+ | Helm is a package manager for Kubernetes and is required to install all the other applications. It will be automatically installed as a dependency when you try to install a different app. It is installed in its own pod inside the cluster which can run the `helm` CLI in a safe environment. | +| [Helm Tiller](https://docs.helm.sh/) | 10.2+ | Helm is a package manager for Kubernetes and is required to install all the other applications. It is installed in its own pod inside the cluster which can run the `helm` CLI in a safe environment. | | [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps] or deploy your own web apps. | -| [Prometheus](https://prometheus.io/docs/introduction/overview/) | 10.4+ | Prometheus is an open-source monitoring and alerting system useful to supervise your deployed applications | +| [Prometheus](https://prometheus.io/docs/introduction/overview/) | 10.4+ | Prometheus is an open-source monitoring and alerting system useful to supervise your deployed applications. | | [GitLab Runner](https://docs.gitlab.com/runner/) | 10.6+ | GitLab Runner is the open source project that is used to run your jobs and send the results back to GitLab. It is used in conjunction with [GitLab CI/CD](https://about.gitlab.com/features/gitlab-ci-cd/), the open-source continuous integration service included with GitLab that coordinates the jobs. When installing the GitLab Runner via the applications, it will run in **privileged mode** by default. Make sure you read the [security implications](#security-implications) before doing so. | | [JupyterHub](http://jupyter.org/) | 11.0+ | [JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) is a multi-user service for managing notebooks across a team. [Jupyter Notebooks](https://jupyter-notebook.readthedocs.io/en/latest/) provide a web-based interactive programming environment used for data analysis, visualization, and machine learning. **Note**: Authentication will be enabled for any user of the GitLab server via OAuth2. HTTPS will be supported in a future release. | diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index a6efe893853..2ec423dcf70 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -1,6 +1,6 @@ # Squash and merge -> [Introduced][ee-1024] in [GitLab Starter][ee] 8.17, and in [GitLab CE][ce] [11.0][ce-18956]. +> [Introduced][ee-1024] in [GitLab Starter][ee] 8.17, and in [GitLab Core][ce] [11.0][ce-18956]. Combine all commits of your merge request into one and retain a clean history. @@ -75,6 +75,6 @@ squashing can itself be considered equivalent to rebasing. [squash-edit-form]: img/squash_edit_form.png [squash-mr-widget]: img/squash_mr_widget.png [ff-merge]: fast_forward_merge.md#enabling-fast-forward-merges -[ce]: https://about.gitlab.com/products/ -[ee]: https://about.gitlab.com/products/ +[ce]: https://about.gitlab.com/pricing/ +[ee]: https://about.gitlab.com/pricing/ [revert]: revert_changes.md diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index f824756c10c..8a2f230f505 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -17,7 +17,7 @@ There are various configuration options to help GitLab server administrators: * Enabling/disabling Git LFS support * Changing the location of LFS object storage -* Setting up AWS S3 compatible object storage +* Setting up object storage supported by [Fog](http://fog.io/about/provider_documentation.html) ### Configuration for Omnibus installations @@ -44,19 +44,31 @@ In `config/gitlab.yml`: storage_path: /mnt/storage/lfs-objects ``` -## Storing the LFS objects in an S3-compatible object storage +## Storing LFS objects in remote object storage > [Introduced][ee-2760] in [GitLab Premium][eep] 10.0. Brought to GitLab Core in 10.7. -It is possible to store LFS objects on a remote object storage which allows you -to offload storage to an external AWS S3 compatible service, freeing up disk -space locally. You can also host your own S3 compatible storage decoupled from -GitLab, with with a service such as [Minio](https://www.minio.io/). +It is possible to store LFS objects in remote object storage which allows you +to offload local hard disk R/W operations, and free up disk space significantly. +GitLab is tightly integrated with `Fog`, so you can refer to its [documentation](http://fog.io/about/provider_documentation.html) +to check which storage services can be integrated with GitLab. +You can also use external object storage in a private local network. For example, +[Minio](https://www.minio.io/) is a standalone object storage service, is easy to setup, and works well with GitLab instances. -Object storage currently transfers files first to GitLab, and then on the -object storage in a second stage. This can be done either by using a rake task -to transfer existing objects, or in a background job after each file is received. +GitLab provides two different options for the uploading mechanism: "Direct upload" and "Background upload". + +**Option 1. Direct upload** + +1. User pushes an lfs file to the GitLab instance +1. GitLab-workhorse uploads the file directly to the external object storage +1. GitLab-workhorse notifies GitLab-rails that the upload process is complete + +**Option 2. Background upload** + +1. User pushes an lfs file to the GitLab instance +1. GitLab-rails stores the file in the local file storage +1. GitLab-rails then uploads the file to the external object storage asynchronously The following general settings are supported. @@ -71,16 +83,50 @@ The following general settings are supported. The `connection` settings match those provided by [Fog](https://github.com/fog). -| Setting | Description | Default | +Here is a configuration example with S3. + +| Setting | Description | example | |---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | AWS | -| `aws_access_key_id` | AWS credentials, or compatible | | -| `aws_secret_access_key` | AWS credentials, or compatible | | +| `provider` | The provider name | AWS | +| `aws_access_key_id` | AWS credentials, or compatible | `ABC123DEF456` | +| `aws_secret_access_key` | AWS credentials, or compatible | `ABC123DEF456ABC123DEF456ABC123DEF456` | | `region` | AWS region | us-east-1 | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | | `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | +Here is a configuration example with GCS. + +| Setting | Description | example | +|---------|-------------|---------| +| `provider` | The provider name | `Google` | +| `google_project` | GCP project name | `gcp-project-12345` | +| `google_client_email` | The email address of the service account | `foo@gcp-project-12345.iam.gserviceaccount.com` | +| `google_json_key_location` | The json key path | `/path/to/gcp-project-12345-abcde.json` | + +_NOTE: The service account must have permission to access the bucket. [See more](https://cloud.google.com/storage/docs/authentication)_ + +### Manual uploading to an object storage + +There are two ways to manually do the same thing as automatic uploading (described above). + +**Option 1: rake task** + +``` +$ rake gitlab:lfs:migrate +``` + +**Option 2: rails console** + +``` +$ sudo gitlab-rails console # Login to rails console + +> # Upload LFS files manually +> LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| +> lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? +> end +``` + ### S3 for Omnibus installations On Omnibus installations, the settings are prefixed by `lfs_object_store_`: @@ -156,6 +202,29 @@ You can see the total storage used for LFS objects on groups and projects in the administration area, as well as through the [groups](../../api/groups.md) and [projects APIs](../../api/projects.md). +## Troubleshooting: `Google::Apis::TransmissionError: execution expired` + +If LFS integration is configred with Google Cloud Storage and background uploads (`background_upload: true` and `direct_upload: false`), +sidekiq workers may encouter this error. This is because the uploading timed out with very large files. +LFS files up to 6Gb can be uploaded without any extra steps, otherwise you need to use the following workaround. + +```shell +$ sudo gitlab-rails console # Login to rails console + +> # Set up timeouts. 20 minutes is enough to upload 30GB LFS files. +> # These settings are only in effect for the same session, i.e. they are not effective for sidekiq workers. +> ::Google::Apis::ClientOptions.default.open_timeout_sec = 1200 +> ::Google::Apis::ClientOptions.default.read_timeout_sec = 1200 +> ::Google::Apis::ClientOptions.default.send_timeout_sec = 1200 + +> # Upload LFS files manually. This process does not use sidekiq at all. +> LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| +> lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? +> end +``` + +See more information in [!19581](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/19581) + ## Known limitations * Support for removing unreferenced LFS objects was added in 8.14 onwards. diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..671b8e7e1b1 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,6 @@ module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c9806cdb85f..341768752dc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -381,15 +381,11 @@ module Gitlab # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end + @repository.gitaly_commit_client.diff_from_parent(self, options) end + # Not to be called directly, but right now its used for tests and in old + # migrations def rugged_diff_from_parent(options = {}) options ||= {} break_rewrites = options[:break_rewrites] diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 8475645971e..5ff15a787f0 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -61,22 +61,15 @@ module Gitlab end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option + cmd = fetch_remote_command(name, tags, prune, force) setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end end - - success end end @@ -202,6 +195,14 @@ module Gitlab private + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') + end + end + def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7056d9c8756..0904e1c2973 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,13 +403,7 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end @@ -613,17 +607,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end # Get refs hash which key is is the commit id @@ -946,13 +930,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -1828,41 +1806,6 @@ module Gitlab commit(sha) end - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index a39b3bc158c..7b2b3bedf04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -22,27 +22,27 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -50,11 +50,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index ccfe0d6bed3..a502ad8a541 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -5,7 +5,7 @@ module Gitlab module RequestForgeryProtection class Controller < ActionController::Base - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true rescue_from ActionController::InvalidAuthenticityToken do |e| logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" diff --git a/rubocop/cop/migration/update_large_table.rb b/rubocop/cop/migration/update_large_table.rb index bb14d0f4f56..c15eec22d04 100644 --- a/rubocop/cop/migration/update_large_table.rb +++ b/rubocop/cop/migration/update_large_table.rb @@ -20,10 +20,14 @@ module RuboCop 'necessary'.freeze LARGE_TABLES = %i[ - ci_pipelines + ci_build_trace_sections ci_builds + ci_job_artifacts + ci_pipelines + ci_stages events issues + merge_request_diff_commits merge_request_diff_files merge_request_diffs merge_requests @@ -34,8 +38,15 @@ module RuboCop users ].freeze + BATCH_UPDATE_METHODS = %w[ + :add_column_with_default + :change_column_type_concurrently + :rename_column_concurrently + :update_column_in_batches + ].join(' ').freeze + def_node_matcher :batch_update?, <<~PATTERN - (send nil? ${:add_column_with_default :update_column_in_batches} $(sym ...) ...) + (send nil? ${#{BATCH_UPDATE_METHODS}} $(sym ...) ...) PATTERN def on_send(node) diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index 2b61e0d4a85..cdec26bd421 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -53,21 +53,22 @@ describe SessionsController do include UserActivitiesHelpers let(:user) { create(:user) } + let(:user_params) { { login: user.username, password: user.password } } it 'authenticates user correctly' do - post(:create, user: { login: user.username, password: user.password }) + post(:create, user: user_params) expect(subject.current_user). to eq user end it 'creates an audit log record' do - expect { post(:create, user: { login: user.username, password: user.password }) }.to change { SecurityEvent.count }.by(1) + expect { post(:create, user: user_params) }.to change { SecurityEvent.count }.by(1) expect(SecurityEvent.last.details[:with]).to eq('standard') end include_examples 'user login request with unique ip limit', 302 do def request - post(:create, user: { login: user.username, password: user.password }) + post(:create, user: user_params) expect(subject.current_user).to eq user subject.sign_out user end @@ -75,10 +76,40 @@ describe SessionsController do it 'updates the user activity' do expect do - post(:create, user: { login: user.username, password: user.password }) + post(:create, user: user_params) end.to change { user_activity(user) } end end + + context 'when reCAPTCHA is enabled' do + let(:user) { create(:user) } + let(:user_params) { { login: user.username, password: user.password } } + + before do + stub_application_setting(recaptcha_enabled: true) + request.headers[described_class::CAPTCHA_HEADER] = 1 + end + + it 'displays an error when the reCAPTCHA is not solved' do + # Without this, `verify_recaptcha` arbitraily returns true in test env + Recaptcha.configuration.skip_verify_env.delete('test') + + post(:create, user: user_params) + + expect(response).to render_template(:new) + expect(flash[:alert]).to include 'There was an error with the reCAPTCHA. Please solve the reCAPTCHA again.' + expect(subject.current_user).to be_nil + end + + it 'successfully logs in a user when reCAPTCHA is solved' do + # Avoid test ordering issue and ensure `verify_recaptcha` returns true + Recaptcha.configuration.skip_verify_env << 'test' + + post(:create, user: user_params) + + expect(subject.current_user).to eq user + end + end end context 'when using two-factor authentication via OTP' do diff --git a/spec/features/admin/admin_groups_spec.rb b/spec/features/admin/admin_groups_spec.rb index d5e603baeae..a4226d7a682 100644 --- a/spec/features/admin/admin_groups_spec.rb +++ b/spec/features/admin/admin_groups_spec.rb @@ -31,6 +31,7 @@ feature 'Admin Groups' do path_component = 'gitlab' group_name = 'GitLab group name' group_description = 'Description of group for GitLab' + fill_in 'group_path', with: path_component fill_in 'group_name', with: group_name fill_in 'group_description', with: group_description diff --git a/spec/features/issues/filtered_search/filter_issues_spec.rb b/spec/features/issues/filtered_search/filter_issues_spec.rb index bc42618306f..8dca81a8627 100644 --- a/spec/features/issues/filtered_search/filter_issues_spec.rb +++ b/spec/features/issues/filtered_search/filter_issues_spec.rb @@ -10,6 +10,7 @@ describe 'Filter issues', :js do # When the name is longer, the filtered search input can end up scrolling # horizontally, and PhantomJS can't handle it. let(:user) { create(:user, name: 'Ann') } + let(:user2) { create(:user, name: 'jane') } let!(:bug_label) { create(:label, project: project, title: 'bug') } let!(:caps_sensitive_label) { create(:label, project: project, title: 'CaPs') } @@ -25,8 +26,6 @@ describe 'Filter issues', :js do before do project.add_master(user) - user2 = create(:user) - create(:issue, project: project, author: user2, title: "Bug report 1") create(:issue, project: project, author: user2, title: "Bug report 2") @@ -113,6 +112,24 @@ describe 'Filter issues', :js do expect_issues_list_count(3) expect_filtered_search_input_empty end + + it 'filters issues by invalid assignee' do + skip('to be tested, issue #26546') + end + + it 'filters issues by multiple assignees' do + create(:issue, project: project, author: user, assignees: [user2, user]) + + input_filtered_search("assignee:@#{user.username} assignee:@#{user2.username}") + + expect_tokens([ + assignee_token(user.name), + assignee_token(user2.name) + ]) + + expect_issues_list_count(1) + expect_filtered_search_input_empty + end end end @@ -491,6 +508,21 @@ describe 'Filter issues', :js do it_behaves_like 'updates atom feed link', :group do let(:path) { issues_group_path(group, milestone_title: milestone.title, assignee_id: user.id) } end + + it 'updates atom feed link for group issues' do + visit issues_group_path(group, milestone_title: milestone.title, assignee_id: user.id) + link = find('.nav-controls a[title="Subscribe to RSS feed"]', visible: false) + params = CGI.parse(URI.parse(link[:href]).query) + auto_discovery_link = find('link[type="application/atom+xml"]', visible: false) + auto_discovery_params = CGI.parse(URI.parse(auto_discovery_link[:href]).query) + + expect(params).to include('feed_token' => [user.feed_token]) + expect(params).to include('milestone_title' => [milestone.title]) + expect(params).to include('assignee_id' => [user.id.to_s]) + expect(auto_discovery_params).to include('feed_token' => [user.feed_token]) + expect(auto_discovery_params).to include('milestone_title' => [milestone.title]) + expect(auto_discovery_params).to include('assignee_id' => [user.id.to_s]) + end end context 'URL has a trailing slash' do diff --git a/spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb index 091645ee86f..7972ff253fe 100644 --- a/spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb @@ -45,7 +45,7 @@ describe Gitlab::Metrics::Samplers::RubySampler do it 'adds a metric containing garbage collection time statistics' do expect(GC::Profiler).to receive(:total_time).and_return(0.24) - expect(sampler.metrics[:total_time]).to receive(:set).with({}, 240) + expect(sampler.metrics[:total_time]).to receive(:increment).with({}, 0.24) sampler.sample end diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index b5a6d959ccb..c5d550cba1b 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -125,14 +125,6 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do end end end - - context 'when data_store is others' do - before do - build_trace_chunk.send(:write_attribute, :data_store, -1) - end - - it { expect { subject }.to raise_error('Unsupported data store') } - end end describe '#truncate' do diff --git a/spec/rubocop/cop/migration/update_large_table_spec.rb b/spec/rubocop/cop/migration/update_large_table_spec.rb index ef724fc8bad..5e08eb4f772 100644 --- a/spec/rubocop/cop/migration/update_large_table_spec.rb +++ b/spec/rubocop/cop/migration/update_large_table_spec.rb @@ -32,6 +32,14 @@ describe RuboCop::Cop::Migration::UpdateLargeTable do include_examples 'large tables', 'add_column_with_default' end + context 'for the change_column_type_concurrently method' do + include_examples 'large tables', 'change_column_type_concurrently' + end + + context 'for the rename_column_concurrently method' do + include_examples 'large tables', 'rename_column_concurrently' + end + context 'for the update_column_in_batches method' do include_examples 'large tables', 'update_column_in_batches' end @@ -60,6 +68,18 @@ describe RuboCop::Cop::Migration::UpdateLargeTable do expect(cop.offenses).to be_empty end + it 'registers no offense for change_column_type_concurrently' do + inspect_source("change_column_type_concurrently :#{table}, :column, default: true") + + expect(cop.offenses).to be_empty + end + + it 'registers no offense for update_column_in_batches' do + inspect_source("rename_column_concurrently :#{table}, :column, default: true") + + expect(cop.offenses).to be_empty + end + it 'registers no offense for update_column_in_batches' do inspect_source("add_column_with_default :#{table}, :column, default: true") diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index da8e660c16b..fce73e0ac1f 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -21,6 +21,11 @@ describe Auth::ContainerRegistryAuthenticationService do allow_any_instance_of(JSONWebToken::RSAToken).to receive(:key).and_return(rsa_key) end + shared_examples 'an authenticated' do + it { is_expected.to include(:token) } + it { expect(payload).to include('access') } + end + shared_examples 'a valid token' do it { is_expected.to include(:token) } it { expect(payload).to include('access') } @@ -380,6 +385,14 @@ describe Auth::ContainerRegistryAuthenticationService do current_project.add_developer(current_user) end + context 'allow to use offline_token' do + let(:current_params) do + { offline_token: true } + end + + it_behaves_like 'an authenticated' + end + it_behaves_like 'a valid token' context 'allow to pull and push images' do diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index 76f1e625fda..f82d4b483e7 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -19,7 +19,9 @@ describe Users::DestroyService do end it 'will delete the project' do - expect_any_instance_of(Projects::DestroyService).to receive(:execute).once + expect_next_instance_of(Projects::DestroyService) do |destroy_service| + expect(destroy_service).to receive(:execute).once + end service.execute(user) end @@ -32,7 +34,9 @@ describe Users::DestroyService do end it 'destroys a project in pending_delete' do - expect_any_instance_of(Projects::DestroyService).to receive(:execute).once + expect_next_instance_of(Projects::DestroyService) do |destroy_service| + expect(destroy_service).to receive(:execute).once + end service.execute(user) diff --git a/spec/support/matchers/match_ids.rb b/spec/support/matchers/match_ids.rb index d8424405b96..1cb6b74acac 100644 --- a/spec/support/matchers/match_ids.rb +++ b/spec/support/matchers/match_ids.rb @@ -10,6 +10,13 @@ RSpec::Matchers.define :match_ids do |*expected| 'matches elements by ids' end + failure_message do + actual_ids = map_ids(actual) + expected_ids = map_ids(expected) + + "expected IDs #{actual_ids} in:\n\n #{actual.inspect}\n\nto match IDs #{expected_ids} in:\n\n #{expected.inspect}" + end + def map_ids(elements) elements = elements.flatten if elements.respond_to?(:flatten) diff --git a/spec/support/redis/redis_shared_examples.rb b/spec/support/redis/redis_shared_examples.rb index 8676f895a83..e650a176041 100644 --- a/spec/support/redis/redis_shared_examples.rb +++ b/spec/support/redis/redis_shared_examples.rb @@ -65,6 +65,14 @@ RSpec.shared_examples "redis_shared_examples" do end describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' unless url1.frozen? + + expect(url2).not_to end_with('foobar') + end + context 'when yml file with env variable' do let(:config_file_name) { config_with_environment_variable_inside } @@ -101,7 +109,6 @@ RSpec.shared_examples "redis_shared_examples" do before do clear_pool end - after do clear_pool end diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb index c7f5694ff43..7e673681c31 100644 --- a/spec/uploaders/object_storage_spec.rb +++ b/spec/uploaders/object_storage_spec.rb @@ -191,6 +191,18 @@ describe ObjectStorage do it "calls a cache path" do expect { |b| uploader.use_file(&b) }.to yield_with_args(%r[tmp/cache]) end + + it "cleans up the cached file" do + cached_path = '' + + uploader.use_file do |path| + cached_path = path + + expect(File.exist?(cached_path)).to be_truthy + end + + expect(File.exist?(cached_path)).to be_falsey + end end end diff --git a/spec/views/devise/shared/_signin_box.html.haml_spec.rb b/spec/views/devise/shared/_signin_box.html.haml_spec.rb index 0870b8f09f9..66c064e3fba 100644 --- a/spec/views/devise/shared/_signin_box.html.haml_spec.rb +++ b/spec/views/devise/shared/_signin_box.html.haml_spec.rb @@ -6,6 +6,7 @@ describe 'devise/shared/_signin_box' do stub_devise assign(:ldap_servers, []) allow(view).to receive(:current_application_settings).and_return(Gitlab::CurrentSettings.current_application_settings) + allow(view).to receive(:captcha_enabled?).and_return(false) end it 'is shown when Crowd is enabled' do |
