diff options
232 files changed, 4482 insertions, 1651 deletions
diff --git a/.gitignore b/.gitignore index 3ffe4263c4f..fcbb4c352a9 100644 --- a/.gitignore +++ b/.gitignore @@ -61,11 +61,9 @@ eslint-report.html /shared/artifacts/ /rails_best_practices_output.html /tags -/tmp/* /vendor/bundle/* /vendor/gitaly-ruby /builds* -/shared/* /.gitlab_workhorse_secret /webpack-report/ /knapsack/ @@ -73,7 +71,6 @@ eslint-report.html /locale/**/LC_MESSAGES /locale/**/*.time_stamp /.rspec -/plugins/* /.gitlab_pages_secret /.gitlab_smime_key /.gitlab_smime_cert diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b5527284d3..b3b6beb1068 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ stages: - prepare - quick-test - test + - review-prepare - review - qa - post-test diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 0720ea3e056..a20215694c0 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -13,7 +13,9 @@ - .default-before_script - .except-docs image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] services: - docker:19.03.0-dind variables: @@ -42,10 +44,6 @@ - time scripts/build_assets_image - scripts/clean-old-cached-assets - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here - # Play dependent manual jobs - - install_api_client_dependencies_with_apt - - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played only: - /.+/@gitlab-org/gitlab-ce - /.+/@gitlab-org/gitlab-ee @@ -186,7 +184,8 @@ jest: - .default-retry - .default-cache - .except-docs - dependencies: ["setup-test-env"] + dependencies: [] + stage: test variables: SETUP_DB: "false" before_script: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 2e8b197829b..878be25c39b 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -165,7 +165,9 @@ downtime_check: - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] ee_compat_check: extends: .rake-exec @@ -195,7 +197,9 @@ db:migrate:reset: - .default-before_script - .use-pg - .except-docs-qa + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] script: - bundle exec rake db:migrate:reset @@ -207,7 +211,9 @@ db:check-schema: - .default-before_script - .use-pg - .except-docs-qa + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] script: - source scripts/schema_changed.sh @@ -219,7 +225,9 @@ db:migrate-from-v11.11.0: - .default-before_script - .use-pg - .except-docs-qa + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: @@ -248,7 +256,9 @@ db:rollback: - .default-before_script - .use-pg - .except-docs-qa + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true @@ -261,7 +271,9 @@ gitlab:setup: - .default-before_script - .use-pg - .except-docs-qa + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 6695404653c..6f1505b5c0d 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -53,8 +53,7 @@ build-qa-image: .review-build-cng-base: image: ruby:2.6-alpine - stage: test - when: manual + stage: review-prepare before_script: - source scripts/utils.sh - install_api_client_dependencies_with_apk @@ -67,17 +66,20 @@ review-build-cng: extends: - .review-build-cng-base - .review-only + needs: ["gitlab:assets:compile pull-cache"] schedule:review-build-cng: extends: - .review-build-cng-base - .review-schedules-only + needs: ["gitlab:assets:compile"] review-deploy: extends: .review-base allow_failure: true retry: 1 stage: review + needs: ["review-build-cng"] variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index d9384780356..cd7ca4d30bf 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -7,7 +7,9 @@ cache gems: - .default-cache - .default-before_script - .except-docs + stage: test dependencies: ["setup-test-env"] + needs: ["setup-test-env"] variables: SETUP_DB: "false" script: diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index 3e107b475c9..dd61cb3f035 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -4,6 +4,7 @@ lint-ci-gitlab: extends: - .default-tags - .default-retry + - .except-docs image: sdesbure/yamllint:latest dependencies: [] script: diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 2ba6b68a53e..5d93758595a 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -17,12 +17,13 @@ Personas can be found at https://about.gitlab.com/handbook/marketing/product-mar ### Permissions and Security -<!-- What permissions are required to perform the described actions? Are they consistent with the existing permissions as documented for users, groups, and projects as appropriate? Is the proposed behavior consistent between the UI, API, and other access methods (e.g. email replies)? --> +<!-- What permissions are required to perform the described actions? Are they consistent with the existing permissions as documented for users, groups, and projects as appropriate? Is the proposed behavior consistent between the UI, API, and other access methods (e.g. email replies)?--> ### Documentation <!-- See the Feature Change Documentation Workflow https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html -Add all known Documentation Requirements here, per https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html#documentation-requirements --> +Add all known Documentation Requirements here, per https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html#documentation-requirements +If this feature requires changing permissions, this document https://docs.gitlab.com/ee/user/permissions.html must be updated accordingly. --> ### Testing @@ -83,7 +83,7 @@ gem 'grape-entity', '~> 0.7.1' gem 'rack-cors', '~> 1.0.0', require: 'rack/cors' # GraphQL API -gem 'graphql', '= 1.8.17' +gem 'graphql', '~> 1.9.11' gem 'graphiql-rails', '~> 1.4.10' gem 'apollo_upload_server', '~> 2.0.0.beta3' gem 'graphql-docs', '~> 1.6.0', group: [:development, :test] @@ -311,7 +311,7 @@ group :metrics do gem 'influxdb', '~> 0.2', require: false # Prometheus - gem 'prometheus-client-mmap', '~> 0.9.8' + gem 'prometheus-client-mmap', '~> 0.9.9' gem 'raindrops', '~> 0.18' end diff --git a/Gemfile.lock b/Gemfile.lock index 451348af280..6add217bc32 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -377,7 +377,7 @@ GEM graphiql-rails (1.4.10) railties sprockets-rails - graphql (1.8.17) + graphql (1.9.11) graphql-docs (1.6.0) commonmarker (~> 0.16) escape_utils (~> 1.2) @@ -650,7 +650,7 @@ GEM parser unparser procto (0.0.3) - prometheus-client-mmap (0.9.8) + prometheus-client-mmap (0.9.9) pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) @@ -1110,7 +1110,7 @@ DEPENDENCIES grape-path-helpers (~> 1.1) grape_logging (~> 1.7) graphiql-rails (~> 1.4.10) - graphql (= 1.8.17) + graphql (~> 1.9.11) graphql-docs (~> 1.6.0) grpc (~> 1.19.0) haml_lint (~> 0.31.0) @@ -1170,7 +1170,7 @@ DEPENDENCIES org-ruby (~> 0.9.12) pg (~> 1.1) premailer-rails (~> 1.9.7) - prometheus-client-mmap (~> 0.9.8) + prometheus-client-mmap (~> 0.9.9) pry-byebug (~> 3.5.1) pry-rails (~> 0.3.4) puma (~> 3.12) diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue index 970f5a7b297..b6da572b201 100644 --- a/app/assets/javascripts/clusters/components/applications.vue +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -397,7 +397,6 @@ export default { </div> </application-row> <application-row - v-if="isProjectCluster" id="jupyter" :logo-url="jupyterhubLogo" :title="applications.jupyter.title" diff --git a/app/assets/javascripts/error_tracking/components/error_tracking_list.vue b/app/assets/javascripts/error_tracking/components/error_tracking_list.vue index 43ae54133af..b1d568532a6 100644 --- a/app/assets/javascripts/error_tracking/components/error_tracking_list.vue +++ b/app/assets/javascripts/error_tracking/components/error_tracking_list.vue @@ -38,6 +38,10 @@ export default { type: String, required: true, }, + userCanEnableErrorTracking: { + type: Boolean, + required: true, + }, }, computed: { ...mapState(['errors', 'externalUrl', 'loading']), @@ -111,14 +115,26 @@ export default { </gl-table> </div> </div> - <div v-else> + <div v-else-if="userCanEnableErrorTracking"> <gl-empty-state :title="__('Get started with error tracking')" - :description="__('Monitor your errors by integrating with Sentry')" + :description="__('Monitor your errors by integrating with Sentry.')" :primary-button-text="__('Enable error tracking')" :primary-button-link="enableErrorTrackingLink" :svg-path="illustrationPath" /> </div> + <div v-else> + <gl-empty-state :title="__('Get started with error tracking')" :svg-path="illustrationPath"> + <template #description> + <div> + <span>{{ __('Monitor your errors by integrating with Sentry.') }}</span> + <a href="/help/user/project/operations/error_tracking.html"> + {{ __('More information') }} + </a> + </div> + </template> + </gl-empty-state> + </div> </div> </template> diff --git a/app/assets/javascripts/error_tracking/index.js b/app/assets/javascripts/error_tracking/index.js index 3d609448efe..073e2c8f1c7 100644 --- a/app/assets/javascripts/error_tracking/index.js +++ b/app/assets/javascripts/error_tracking/index.js @@ -14,9 +14,10 @@ export default () => { render(createElement) { const domEl = document.querySelector(this.$options.el); const { indexPath, enableErrorTrackingLink, illustrationPath } = domEl.dataset; - let { errorTrackingEnabled } = domEl.dataset; + let { errorTrackingEnabled, userCanEnableErrorTracking } = domEl.dataset; errorTrackingEnabled = parseBoolean(errorTrackingEnabled); + userCanEnableErrorTracking = parseBoolean(userCanEnableErrorTracking); return createElement('error-tracking-list', { props: { @@ -24,6 +25,7 @@ export default () => { enableErrorTrackingLink, errorTrackingEnabled, illustrationPath, + userCanEnableErrorTracking, }, }); }, diff --git a/app/assets/javascripts/jobs/components/job_app.vue b/app/assets/javascripts/jobs/components/job_app.vue index ad1072366f3..c7d4d7c4b9b 100644 --- a/app/assets/javascripts/jobs/components/job_app.vue +++ b/app/assets/javascripts/jobs/components/job_app.vue @@ -83,6 +83,11 @@ export default { type: String, required: true, }, + subscriptionsMoreMinutesUrl: { + type: String, + required: false, + default: null, + }, }, computed: { ...mapState([ @@ -265,6 +270,7 @@ export default { :quota-limit="job.runners.quota.limit" :runners-path="runnerHelpUrl" :project-path="projectPath" + :subscriptions-more-minutes-url="subscriptionsMoreMinutesUrl" /> <environments-block diff --git a/app/assets/javascripts/jobs/index.js b/app/assets/javascripts/jobs/index.js index add7f9b710a..9c35534523e 100644 --- a/app/assets/javascripts/jobs/index.js +++ b/app/assets/javascripts/jobs/index.js @@ -15,6 +15,7 @@ export default () => { runnerHelpUrl, runnerSettingsUrl, variablesSettingsUrl, + subscriptionsMoreMinutesUrl, endpoint, pagePath, logState, @@ -28,6 +29,7 @@ export default () => { runnerHelpUrl, runnerSettingsUrl, variablesSettingsUrl, + subscriptionsMoreMinutesUrl, endpoint, pagePath, logState, diff --git a/app/assets/javascripts/lib/utils/axios_utils.js b/app/assets/javascripts/lib/utils/axios_utils.js index 69159e2d741..37721cd030c 100644 --- a/app/assets/javascripts/lib/utils/axios_utils.js +++ b/app/assets/javascripts/lib/utils/axios_utils.js @@ -10,21 +10,18 @@ axios.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest'; axios.interceptors.request.use(config => { window.activeVueResources = window.activeVueResources || 0; window.activeVueResources += 1; - return config; }); // Remove the global counter axios.interceptors.response.use( - config => { + response => { window.activeVueResources -= 1; - - return config; + return response; }, - e => { + err => { window.activeVueResources -= 1; - - return Promise.reject(e); + return Promise.reject(err); }, ); diff --git a/app/assets/javascripts/pages/projects/project.js b/app/assets/javascripts/pages/projects/project.js index 332b6811af6..33e9a8e9d56 100644 --- a/app/assets/javascripts/pages/projects/project.js +++ b/app/assets/javascripts/pages/projects/project.js @@ -73,13 +73,6 @@ export default class Project { .remove(); return e.preventDefault(); }); - $('.hide-shared-runner-limit-message').on('click', function(e) { - var $alert = $(this).parents('.shared-runner-quota-message'); - var scope = $alert.data('scope'); - Cookies.set('hide_shared_runner_quota_message', 'false', { path: scope }); - $alert.remove(); - e.preventDefault(); - }); $('.hide-auto-devops-implicitly-enabled-banner').on('click', function(e) { const projectId = $(this).data('project-id'); const cookieKey = `hide_auto_devops_implicitly_enabled_banner_${projectId}`; diff --git a/app/controllers/projects/settings/operations_controller.rb b/app/controllers/projects/settings/operations_controller.rb index 5cfb0ac307d..ec89bb89edc 100644 --- a/app/controllers/projects/settings/operations_controller.rb +++ b/app/controllers/projects/settings/operations_controller.rb @@ -3,7 +3,7 @@ module Projects module Settings class OperationsController < Projects::ApplicationController - before_action :authorize_update_environment! + before_action :authorize_admin_operations! helper_method :error_tracking_setting diff --git a/app/graphql/types/permission_types/project.rb b/app/graphql/types/permission_types/project.rb index e9a4ea9157b..993d33c4fc2 100644 --- a/app/graphql/types/permission_types/project.rb +++ b/app/graphql/types/permission_types/project.rb @@ -16,7 +16,7 @@ module Types :create_deployment, :push_to_delete_protected_branch, :admin_wiki, :admin_project, :update_pages, :admin_remote_mirror, :create_label, :update_wiki, :destroy_wiki, - :create_pages, :destroy_pages, :read_pages_content + :create_pages, :destroy_pages, :read_pages_content, :admin_operations end end end diff --git a/app/helpers/external_link_helper.rb b/app/helpers/external_link_helper.rb new file mode 100644 index 00000000000..9dbad1f5032 --- /dev/null +++ b/app/helpers/external_link_helper.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module ExternalLinkHelper + def external_link(body, url, options = {}) + link_to url, { target: '_blank', rel: 'noopener noreferrer' }.merge(options) do + "#{body} #{icon('external-link')}".html_safe + end + end +end diff --git a/app/helpers/jobs_helper.rb b/app/helpers/jobs_helper.rb new file mode 100644 index 00000000000..46edba261dd --- /dev/null +++ b/app/helpers/jobs_helper.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module JobsHelper + def jobs_data + { + "endpoint" => project_job_path(@project, @build, format: :json), + "project_path" => @project.full_path, + "deployment_help_url" => help_page_path('user/project/clusters/index.html', anchor: 'troubleshooting-failed-deployment-jobs'), + "runner_help_url" => help_page_path('ci/runners/README.html', anchor: 'setting-maximum-job-timeout-for-a-runner'), + "runner_settings_url" => project_runners_path(@build.project, anchor: 'js-runners-settings'), + "variables_settings_url" => project_variables_path(@build.project, anchor: 'js-cicd-variables-settings'), + "page_path" => project_job_path(@project, @build), + "build_status" => @build.status, + "build_stage" => @build.stage, + "log_state" => '', + "build_options" => javascript_build_options + } + end +end diff --git a/app/helpers/projects/error_tracking_helper.rb b/app/helpers/projects/error_tracking_helper.rb index 6daf2e21ca2..fd1222a1dfb 100644 --- a/app/helpers/projects/error_tracking_helper.rb +++ b/app/helpers/projects/error_tracking_helper.rb @@ -1,12 +1,13 @@ # frozen_string_literal: true module Projects::ErrorTrackingHelper - def error_tracking_data(project) + def error_tracking_data(current_user, project) error_tracking_enabled = !!project.error_tracking_setting&.enabled? { 'index-path' => project_error_tracking_index_path(project, format: :json), + 'user-can-enable-error-tracking' => can?(current_user, :admin_operations, project).to_s, 'enable-error-tracking-link' => project_settings_operations_path(project), 'error-tracking-enabled' => error_tracking_enabled.to_s, 'illustration-path' => image_path('illustrations/cluster_popover.svg') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d558f66154e..72782827906 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -88,6 +88,7 @@ module Ci validates :coverage, numericality: true, allow_blank: true validates :ref, presence: true + scope :not_interruptible, -> { joins(:metadata).where(ci_builds_metadata: { interruptible: false }) } scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } scope :with_artifacts_archive, ->() do diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2b6f10ef79f..d620959b538 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -225,6 +225,14 @@ module Ci where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) end + scope :without_interruptible_builds, -> do + where('NOT EXISTS (?)', + Ci::Build.where('ci_builds.commit_id = ci_pipelines.id') + .with_status(:running, :success, :failed) + .not_interruptible + ) + end + # Returns the pipelines in descending order (= newest first), optionally # limited to a number of references. # diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index fb74d96efe3..ec65482a846 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -85,7 +85,8 @@ module Clusters "clientId" => oauth_application.uid, "clientSecret" => oauth_application.secret, "callbackUrl" => callback_url, - "gitlabProjectIdWhitelist" => [project_id] + "gitlabProjectIdWhitelist" => cluster.projects.ids, + "gitlabGroupWhitelist" => cluster.groups.map(&:to_param) } }, "singleuser" => { @@ -101,10 +102,6 @@ module Clusters @crypto_key ||= SecureRandom.hex(32) end - def project_id - cluster&.project&.id - end - def gitlab_url Gitlab.config.gitlab.url end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 444e1a82c97..ef1af1fc8bc 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -10,15 +10,15 @@ module Clusters self.table_name = 'clusters' PROJECT_ONLY_APPLICATIONS = { - Applications::Jupyter.application_name => Applications::Jupyter, Applications::Knative.application_name => Applications::Knative }.freeze APPLICATIONS = { Applications::Helm.application_name => Applications::Helm, Applications::Ingress.application_name => Applications::Ingress, Applications::CertManager.application_name => Applications::CertManager, + Applications::Prometheus.application_name => Applications::Prometheus, Applications::Runner.application_name => Applications::Runner, - Applications::Prometheus.application_name => Applications::Prometheus + Applications::Jupyter.application_name => Applications::Jupyter }.merge(PROJECT_ONLY_APPLICATIONS).freeze DEFAULT_ENVIRONMENT = '*' KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 304cc71e9dc..a0ca8a34c6d 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -15,6 +15,7 @@ module Ci autosave: true delegate :timeout, to: :metadata, prefix: true, allow_nil: true + delegate :interruptible, to: :metadata, prefix: false, allow_nil: true before_create :ensure_metadata end @@ -50,6 +51,14 @@ module Ci write_metadata_attribute(:yaml_variables, :config_variables, value) end + def interruptible + metadata&.interruptible + end + + def interruptible=(value) + ensure_metadata.interruptible = value + end + private def read_metadata_attribute(legacy_key, metadata_key, default_value = nil) diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index cf88076ac74..bcbbb27a9a8 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -102,6 +102,7 @@ module HasStatus scope :manual, -> { with_status(:manual) } scope :scheduled, -> { with_status(:scheduled) } scope :alive, -> { with_status(:created, :preparing, :pending, :running) } + scope :alive_or_scheduled, -> { with_status(:created, :preparing, :pending, :running, :scheduled) } scope :created_or_pending, -> { with_status(:created, :pending) } scope :running_or_pending, -> { with_status(:running, :pending) } scope :finished, -> { with_status(:success, :failed, :canceled) } diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 07d22641a0a..8b011bca72c 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -33,7 +33,7 @@ module Routable # # Returns a single object, or nil. def find_by_full_path(path, follow_redirects: false) - increment_counter(:routable_find_by_full_path, 'Number of calls to Routable.find_by_full_path') + routable_calls_counter.increment(method: 'find_by_full_path') if Feature.enabled?(:routable_two_step_lookup) # Case sensitive match first (it's cheaper and the usual case) @@ -61,7 +61,7 @@ module Routable def where_full_path_in(paths) return none if paths.empty? - increment_counter(:routable_where_full_path_in, 'Number of calls to Routable.where_full_path_in') + routable_calls_counter.increment(method: 'where_full_path_in') wheres = paths.map do |path| "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" @@ -71,12 +71,8 @@ module Routable end # Temporary instrumentation of method calls - def increment_counter(counter, description) - @counters[counter] ||= Gitlab::Metrics.counter(counter, description) - - @counters[counter].increment - rescue - # ignore the error + def routable_calls_counter + @routable_calls_counter ||= Gitlab::Metrics.counter(:gitlab_routable_calls_total, 'Number of calls to Routable by method') end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 28e450f9b30..95daa48d4bc 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -174,6 +174,7 @@ class MergeRequest < ApplicationRecord scope :from_project, ->(project) { where(source_project_id: project.id) } scope :merged, -> { with_state(:merged) } scope :closed_and_merged, -> { with_states(:closed, :merged) } + scope :open_and_closed, -> { with_states(:opened, :closed) } scope :from_source_branches, ->(branches) { where(source_branch: branches) } scope :by_commit_sha, ->(sha) do where('EXISTS (?)', MergeRequestDiff.select(1).where('merge_requests.latest_merge_request_diff_id = merge_request_diffs.id').by_commit_sha(sha)).reorder(nil) @@ -187,6 +188,11 @@ class MergeRequest < ApplicationRecord target_project: [:route, { namespace: :route }], source_project: [:route, { namespace: :route }]) } + scope :by_target_branch_wildcard, ->(wildcard_branch_name) do + where("target_branch LIKE ?", ApplicationRecord.sanitize_sql_like(wildcard_branch_name).tr('*', '%')) + end + scope :by_target_branch, ->(branch_name) { where(target_branch: branch_name) } + scope :preload_source_project, -> { preload(:source_project) } after_save :keep_around_commit @@ -1232,9 +1238,9 @@ class MergeRequest < ApplicationRecord compare_reports(Ci::CompareTestReportsService) end - def compare_reports(service_class) + def compare_reports(service_class, current_user = nil) with_reactive_cache(service_class.name) do |data| - unless service_class.new(project) + unless service_class.new(project, current_user) .latest?(base_pipeline, actual_head_pipeline, data) raise InvalidateReactiveCache end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index ee0c94c20af..9fd929371f8 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -3,6 +3,9 @@ class ProtectedBranch < ApplicationRecord include ProtectedRef + scope :requiring_code_owner_approval, + -> { where(code_owner_approval_required: true) } + protected_ref_access_levels :merge, :push def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index e2634692dc7..5c36b59f07b 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -294,6 +294,7 @@ class ProjectPolicy < BasePolicy enable :destroy_release enable :destroy_artifacts enable :daily_statistics + enable :admin_operations end rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 29317f1176e..8f8582afb43 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -91,11 +91,21 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - project.ci_pipelines - .where(ref: pipeline.ref) - .where.not(id: pipeline.id) - .where.not(sha: project.commit(pipeline.ref).try(:id)) - .created_or_pending + # TODO: Introduced by https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23464 + if Feature.enabled?(:ci_support_interruptible_pipelines, project, default_enabled: true) + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.id) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .without_interruptible_builds + else + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.id) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .created_or_pending + end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb index 65d08966802..1ce6e0c1cb0 100644 --- a/app/services/clusters/applications/check_installation_progress_service.rb +++ b/app/services/clusters/applications/check_installation_progress_service.rb @@ -33,6 +33,10 @@ module Clusters def timed_out? Time.now.utc - app.updated_at.utc > ClusterWaitForAppInstallationWorker::TIMEOUT end + + def remove_installation_pod + helm_api.delete_pod!(pod_name) + end end end end diff --git a/app/services/clusters/applications/check_uninstall_progress_service.rb b/app/services/clusters/applications/check_uninstall_progress_service.rb index 6a618d61c4f..fe9c488bdfd 100644 --- a/app/services/clusters/applications/check_uninstall_progress_service.rb +++ b/app/services/clusters/applications/check_uninstall_progress_service.rb @@ -15,7 +15,7 @@ module Clusters rescue StandardError => e app.make_errored!(_('Application uninstalled but failed to destroy: %{error_message}') % { error_message: e.message }) ensure - remove_installation_pod + remove_uninstallation_pod end def check_timeout @@ -33,6 +33,10 @@ module Clusters def timed_out? Time.now.utc - app.updated_at.utc > WaitForUninstallAppWorker::TIMEOUT end + + def remove_uninstallation_pod + helm_api.delete_pod!(pod_name) + end end end end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 3555864f834..900e5063621 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -76,13 +76,16 @@ class IssuableBaseService < BaseService end def filter_labels - params[:add_label_ids] = labels_service.filter_labels_ids_in_param(:add_label_ids) if params[:add_label_ids] - params[:remove_label_ids] = labels_service.filter_labels_ids_in_param(:remove_label_ids) if params[:remove_label_ids] + label_ids_to_filter(:add_label_ids, :add_labels, false) + label_ids_to_filter(:remove_label_ids, :remove_labels, true) + label_ids_to_filter(:label_ids, :labels, false) + end - if params[:label_ids] - params[:label_ids] = labels_service.filter_labels_ids_in_param(:label_ids) - elsif params[:labels] - params[:label_ids] = labels_service.find_or_create_by_titles.map(&:id) + def label_ids_to_filter(label_id_key, label_key, find_only) + if params[label_id_key] + params[label_id_key] = labels_service.filter_labels_ids_in_param(label_id_key) + elsif params[label_key] + params[label_id_key] = labels_service.find_or_create_by_titles(label_key, find_only: find_only).map(&:id) end end diff --git a/app/services/labels/available_labels_service.rb b/app/services/labels/available_labels_service.rb index fe477d96970..8886e58d6ef 100644 --- a/app/services/labels/available_labels_service.rb +++ b/app/services/labels/available_labels_service.rb @@ -9,8 +9,8 @@ module Labels @params = params end - def find_or_create_by_titles - labels = params.delete(:labels) + def find_or_create_by_titles(key = :labels, find_only: false) + labels = params.delete(key) return [] unless labels @@ -23,7 +23,7 @@ module Labels include_ancestor_groups: true, title: label_name.strip, available_labels: available_labels - ).execute + ).execute(find_only: find_only) label end.compact diff --git a/app/services/labels/find_or_create_service.rb b/app/services/labels/find_or_create_service.rb index 628873519d7..a47dd42aea0 100644 --- a/app/services/labels/find_or_create_service.rb +++ b/app/services/labels/find_or_create_service.rb @@ -9,9 +9,9 @@ module Labels @params = params.dup.with_indifferent_access end - def execute(skip_authorization: false) + def execute(skip_authorization: false, find_only: false) @skip_authorization = skip_authorization - find_or_create_label + find_or_create_label(find_only: find_only) end private @@ -30,9 +30,11 @@ module Labels # Only creates the label if current_user can do so, if the label does not exist # and the user can not create the label, nil is returned # rubocop: disable CodeReuse/ActiveRecord - def find_or_create_label + def find_or_create_label(find_only: false) new_label = available_labels.find_by(title: title) + return new_label if find_only + if new_label.nil? && (skip_authorization || Ability.allowed?(current_user, :admin_label, parent)) create_params = params.except(:include_ancestor_groups) new_label = Labels::CreateService.new(create_params).execute(parent_type.to_sym => parent) diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb index b28f80939ae..308a3a10d1a 100644 --- a/app/services/merge_requests/build_service.rb +++ b/app/services/merge_requests/build_service.rb @@ -18,6 +18,18 @@ module MergeRequests merge_request.target_project = find_target_project filter_params(merge_request) + + # merge_request.assign_attributes(...) below is a Rails + # method that only work if all the params it is passed have + # corresponding fields in the database. As there are no fields + # in the database for :add_label_ids and :remove_label_ids, we + # need to remove them from the params before the call to + # merge_request.assign_attributes(...) + # + # IssuableBaseService#process_label_ids takes care + # of the removal. + params[:label_ids] = process_label_ids(params, extra_label_ids: merge_request.label_ids.to_a) + merge_request.assign_attributes(params.to_h.compact) merge_request.compare_commits = [] diff --git a/app/services/merge_requests/push_options_handler_service.rb b/app/services/merge_requests/push_options_handler_service.rb index b210004e6e1..0168b31005e 100644 --- a/app/services/merge_requests/push_options_handler_service.rb +++ b/app/services/merge_requests/push_options_handler_service.rb @@ -100,7 +100,8 @@ module MergeRequests merge_request = ::MergeRequests::CreateService.new( project, current_user, - merge_request.attributes.merge(assignees: merge_request.assignees) + merge_request.attributes.merge(assignees: merge_request.assignees, + label_ids: merge_request.label_ids) ).execute end @@ -122,7 +123,9 @@ module MergeRequests title: push_options[:title], description: push_options[:description], target_branch: push_options[:target], - force_remove_source_branch: push_options[:remove_source_branch] + force_remove_source_branch: push_options[:remove_source_branch], + label: push_options[:label], + unlabel: push_options[:unlabel] } params.compact! @@ -134,6 +137,9 @@ module MergeRequests ) end + params[:add_labels] = params.delete(:label).keys if params.has_key?(:label) + params[:remove_labels] = params.delete(:unlabel).keys if params.has_key?(:unlabel) + params end diff --git a/app/services/protected_branches/create_service.rb b/app/services/protected_branches/create_service.rb index 87aaf4672a4..6b2836bba39 100644 --- a/app/services/protected_branches/create_service.rb +++ b/app/services/protected_branches/create_service.rb @@ -5,7 +5,8 @@ module ProtectedBranches def execute(skip_authorization: false) raise Gitlab::Access::AccessDeniedError unless skip_authorization || authorized? - protected_branch.save + save_protected_branch + protected_branch end @@ -15,6 +16,10 @@ module ProtectedBranches private + def save_protected_branch + protected_branch.save + end + def protected_branch @protected_branch ||= project.protected_branches.new(params) end diff --git a/app/validators/addressable_url_validator.rb b/app/validators/addressable_url_validator.rb index bb445499cee..f292730441c 100644 --- a/app/validators/addressable_url_validator.rb +++ b/app/validators/addressable_url_validator.rb @@ -42,6 +42,11 @@ class AddressableUrlValidator < ActiveModel::EachValidator attr_reader :record + # By default, we avoid checking the dns rebinding protection + # when saving/updating a record. Sometimes, the url + # is not resolvable at that point, and some automated + # tasks that uses that url won't work. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/66723 BLOCKER_VALIDATE_OPTIONS = { schemes: %w(http https), ports: [], @@ -49,7 +54,8 @@ class AddressableUrlValidator < ActiveModel::EachValidator allow_local_network: true, ascii_only: false, enforce_user: false, - enforce_sanitization: false + enforce_sanitization: false, + dns_rebind_protection: false }.freeze DEFAULT_OPTIONS = BLOCKER_VALIDATE_OPTIONS.merge({ diff --git a/app/views/projects/error_tracking/index.html.haml b/app/views/projects/error_tracking/index.html.haml index bc02c5f0e5a..96f61584a99 100644 --- a/app/views/projects/error_tracking/index.html.haml +++ b/app/views/projects/error_tracking/index.html.haml @@ -1,3 +1,3 @@ - page_title _('Errors') -#js-error_tracking{ data: error_tracking_data(@project) } +#js-error_tracking{ data: error_tracking_data(@current_user, @project) } diff --git a/app/views/projects/jobs/show.html.haml b/app/views/projects/jobs/show.html.haml index 6bb27a65142..2e322c7db23 100644 --- a/app/views/projects/jobs/show.html.haml +++ b/app/views/projects/jobs/show.html.haml @@ -5,10 +5,4 @@ - content_for :page_specific_javascripts do = stylesheet_link_tag 'page_bundles/xterm' -#js-job-vue-app{ data: { endpoint: project_job_path(@project, @build, format: :json), project_path: @project.full_path, - deployment_help_url: help_page_path('user/project/clusters/index.html', anchor: 'troubleshooting-failed-deployment-jobs'), - runner_help_url: help_page_path('ci/runners/README.html', anchor: 'setting-maximum-job-timeout-for-a-runner'), - runner_settings_url: project_runners_path(@build.project, anchor: 'js-runners-settings'), - variables_settings_url: project_variables_path(@build.project, anchor: 'js-cicd-variables-settings'), - page_path: project_job_path(@project, @build), build_status: @build.status, build_stage: @build.stage, log_state: '', - build_options: javascript_build_options } } +#js-job-vue-app{ data: jobs_data } diff --git a/app/views/projects/mirrors/_instructions.html.haml b/app/views/projects/mirrors/_instructions.html.haml index 1a163cc4a54..7ff6c0a2019 100644 --- a/app/views/projects/mirrors/_instructions.html.haml +++ b/app/views/projects/mirrors/_instructions.html.haml @@ -3,6 +3,7 @@ %li = _('The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.').html_safe + %li= _('When using the <code>http://</code> or <code>https://</code> protocols, please provide the exact URL to the repository. HTTP redirects will not be followed.').html_safe %li= _('Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.').html_safe %li - minutes = Gitlab.config.gitlab_shell.git_timeout / 60 diff --git a/app/views/projects/pages/_access.html.haml b/app/views/projects/pages/_access.html.haml index 539f223ca9b..7b6d46964a2 100644 --- a/app/views/projects/pages/_access.html.haml +++ b/app/views/projects/pages/_access.html.haml @@ -7,9 +7,11 @@ %strong = _("Your pages are served under:") - %p= link_to @project.pages_url, @project.pages_url + %p + = external_link(@project.pages_url, @project.pages_url) - @project.pages_domains.each do |domain| - %p= link_to domain.url, domain.url + %p + = external_link(domain.url, domain.url) .card-footer.alert-primary = _("It may take up to 30 minutes before the site is available after the first deployment.") diff --git a/app/views/projects/pages/_list.html.haml b/app/views/projects/pages/_list.html.haml index 2427b4d7611..c4285e7f3d2 100644 --- a/app/views/projects/pages/_list.html.haml +++ b/app/views/projects/pages/_list.html.haml @@ -12,9 +12,7 @@ .domain-status.ci-status-icon.has-tooltip{ class: "ci-status-icon-#{status}", title: tooltip } = sprite_icon("status_#{status}", size: 16 ) .domain-name - = link_to domain.url do - = domain.url - = icon('external-link') + = external_link(domain.url, domain.url) - if domain.subject %div %span.badge.badge-gray Certificate: #{domain.subject} diff --git a/app/views/projects/pages_domains/show.html.haml b/app/views/projects/pages_domains/show.html.haml index d0b54946f7e..33837e21c8d 100644 --- a/app/views/projects/pages_domains/show.html.haml +++ b/app/views/projects/pages_domains/show.html.haml @@ -21,9 +21,7 @@ %td = _("Domain") %td - = link_to @domain.url do - = @domain.url - = icon('external-link') + = external_link(@domain.url, @domain.url) %tr %td = _("DNS") diff --git a/app/views/shared/_import_form.html.haml b/app/views/shared/_import_form.html.haml index d0f9374e832..b2ea45d6f1a 100644 --- a/app/views/shared/_import_form.html.haml +++ b/app/views/shared/_import_form.html.haml @@ -27,6 +27,7 @@ %ul %li = _('The repository must be accessible over <code>http://</code>, <code>https://</code> or <code>git://</code>.').html_safe + %li= _('When using the <code>http://</code> or <code>https://</code> protocols, please provide the exact URL to the repository. HTTP redirects will not be followed.').html_safe %li = _('If your HTTP repository is not publicly accessible, add your credentials.') %li diff --git a/changelogs/unreleased/64009-show-a-meaningful-error-message-when-due-quick_actions-command-fail.yml b/changelogs/unreleased/64009-show-a-meaningful-error-message-when-due-quick_actions-command-fail.yml new file mode 100644 index 00000000000..8624b868686 --- /dev/null +++ b/changelogs/unreleased/64009-show-a-meaningful-error-message-when-due-quick_actions-command-fail.yml @@ -0,0 +1,5 @@ +--- +title: Show meaningful message on /due quick action with invalid date +merge_request: 32349 +author: Jacopo Beschi @jacopo-beschi +type: changed diff --git a/changelogs/unreleased/66067-pages-domain-doesnt-set-target-blank.yml b/changelogs/unreleased/66067-pages-domain-doesnt-set-target-blank.yml new file mode 100644 index 00000000000..726d4b163d2 --- /dev/null +++ b/changelogs/unreleased/66067-pages-domain-doesnt-set-target-blank.yml @@ -0,0 +1,5 @@ +--- +title: Makes custom Pages domain open as external link in new tab +merge_request: 32130 +author: jakeburden +type: fixed diff --git a/changelogs/unreleased/66150-remove-dynamically-constructed-feature-flags-starting-with-promethe.yml b/changelogs/unreleased/66150-remove-dynamically-constructed-feature-flags-starting-with-promethe.yml new file mode 100644 index 00000000000..e6dc1f1bec6 --- /dev/null +++ b/changelogs/unreleased/66150-remove-dynamically-constructed-feature-flags-starting-with-promethe.yml @@ -0,0 +1,5 @@ +--- +title: Remove dynamically constructed feature flags starting with prometheus_transaction_ +merge_request: 32395 +author: Jacopo Beschi @jacopo-beschi +type: changed diff --git a/changelogs/unreleased/66467-enable-error-tracking-only-user-can-read-sentry-logs.yml b/changelogs/unreleased/66467-enable-error-tracking-only-user-can-read-sentry-logs.yml new file mode 100644 index 00000000000..b152943942f --- /dev/null +++ b/changelogs/unreleased/66467-enable-error-tracking-only-user-can-read-sentry-logs.yml @@ -0,0 +1,5 @@ +--- +title: Display `more information` docs link on error tracking page when users do not have permissions to enable that feature +merge_request: 32365 +author: Romain Maneschi +type: fixed diff --git a/changelogs/unreleased/67037-user-content-gitlab-static-net-brings-back-404-only.yml b/changelogs/unreleased/67037-user-content-gitlab-static-net-brings-back-404-only.yml new file mode 100644 index 00000000000..8d4f816af2e --- /dev/null +++ b/changelogs/unreleased/67037-user-content-gitlab-static-net-brings-back-404-only.yml @@ -0,0 +1,5 @@ +--- +title: Default the asset proxy whitelist to the installation domain +merge_request: 32703 +author: +type: fixed diff --git a/changelogs/unreleased/ab-add-index-for-ci-builds-metrics.yml b/changelogs/unreleased/ab-add-index-for-ci-builds-metrics.yml new file mode 100644 index 00000000000..03a37dc0c04 --- /dev/null +++ b/changelogs/unreleased/ab-add-index-for-ci-builds-metrics.yml @@ -0,0 +1,5 @@ +--- +title: Create partial index for gitlab-monitor CI metrics +merge_request: 32546 +author: +type: performance diff --git a/changelogs/unreleased/ab-unconfirmed-email-index.yml b/changelogs/unreleased/ab-unconfirmed-email-index.yml new file mode 100644 index 00000000000..3887cd87e41 --- /dev/null +++ b/changelogs/unreleased/ab-unconfirmed-email-index.yml @@ -0,0 +1,5 @@ +--- +title: Create index for users.unconfirmed_email +merge_request: 32664 +author: +type: performance diff --git a/changelogs/unreleased/add-label-push-opts.yml b/changelogs/unreleased/add-label-push-opts.yml new file mode 100644 index 00000000000..1289020e4e5 --- /dev/null +++ b/changelogs/unreleased/add-label-push-opts.yml @@ -0,0 +1,5 @@ +--- +title: Support adding and removing labels w/ push opts +merge_request: 31831 +author: +type: added diff --git a/changelogs/unreleased/fix-regression-remove-installation-pod.yml b/changelogs/unreleased/fix-regression-remove-installation-pod.yml new file mode 100644 index 00000000000..1ed72f1189d --- /dev/null +++ b/changelogs/unreleased/fix-regression-remove-installation-pod.yml @@ -0,0 +1,5 @@ +--- +title: Fix removal of install pods +merge_request: 32667 +author: +type: fixed diff --git a/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml b/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml new file mode 100644 index 00000000000..c1372a4a73e --- /dev/null +++ b/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml @@ -0,0 +1,5 @@ +--- +title: Allow not resolvable urls when dns rebind protection is disabled +merge_request: 32523 +author: +type: fixed diff --git a/changelogs/unreleased/fj-remove-dns-protection-when-validating.yml b/changelogs/unreleased/fj-remove-dns-protection-when-validating.yml new file mode 100644 index 00000000000..9c74f8d69c7 --- /dev/null +++ b/changelogs/unreleased/fj-remove-dns-protection-when-validating.yml @@ -0,0 +1,5 @@ +--- +title: Avoid checking dns rebind protection when validating +merge_request: 32577 +author: +type: fixed diff --git a/changelogs/unreleased/group_level_jupyterhub.yml b/changelogs/unreleased/group_level_jupyterhub.yml new file mode 100644 index 00000000000..81fc7600e0e --- /dev/null +++ b/changelogs/unreleased/group_level_jupyterhub.yml @@ -0,0 +1,5 @@ +--- +title: Group level JupyterHub +merge_request: 32512 +author: +type: added diff --git a/changelogs/unreleased/issue-32741.yml b/changelogs/unreleased/issue-32741.yml new file mode 100644 index 00000000000..5d0a3ee2cc7 --- /dev/null +++ b/changelogs/unreleased/issue-32741.yml @@ -0,0 +1,5 @@ +--- +title: New interruptible attribute for CI/CD jobs +merge_request: 23464 +author: Cédric Tabin +type: added diff --git a/config/routes.rb b/config/routes.rb index 02a405a91f8..a622ce268da 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -107,6 +107,7 @@ Rails.application.routes.draw do draw :instance_statistics Gitlab.ee do + draw :security draw :smartcard draw :jira_connect draw :username diff --git a/db/migrate/20190729180447_add_merge_requests_require_code_owner_approval_to_protected_branches.rb b/db/migrate/20190729180447_add_merge_requests_require_code_owner_approval_to_protected_branches.rb new file mode 100644 index 00000000000..098fcff9ace --- /dev/null +++ b/db/migrate/20190729180447_add_merge_requests_require_code_owner_approval_to_protected_branches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class AddMergeRequestsRequireCodeOwnerApprovalToProtectedBranches < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default( + :protected_branches, + :code_owner_approval_required, + :boolean, + default: false + ) + + add_concurrent_index( + :protected_branches, + [:project_id, :code_owner_approval_required], + name: "code_owner_approval_required", + where: "code_owner_approval_required = #{Gitlab::Database.true_value}") + end + + def down + remove_concurrent_index(:protected_branches, name: "code_owner_approval_required") + + remove_column(:protected_branches, :code_owner_approval_required) + end +end diff --git a/db/migrate/20190902152329_add_index_for_ci_builds_metrics.rb b/db/migrate/20190902152329_add_index_for_ci_builds_metrics.rb new file mode 100644 index 00000000000..b755d60e311 --- /dev/null +++ b/db/migrate/20190902152329_add_index_for_ci_builds_metrics.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexForCiBuildsMetrics < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + INDEX_NAME = 'ci_builds_gitlab_monitor_metrics' + + def up + add_concurrent_index(:ci_builds, [:status, :created_at, :project_id], where: "type = 'Ci::Build'", name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:ci_builds, INDEX_NAME) + end +end diff --git a/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb b/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb new file mode 100644 index 00000000000..e78d47f023f --- /dev/null +++ b/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexOnUsersUnconfirmedEmail < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :users, :unconfirmed_email, where: 'unconfirmed_email IS NOT NULL' + end + + def down + remove_concurrent_index :users, :unconfirmed_email, where: 'unconfirmed_email IS NOT NULL' + end +end diff --git a/db/migrate/20190905223800_add_interruptible_to_builds_metadata.rb b/db/migrate/20190905223800_add_interruptible_to_builds_metadata.rb new file mode 100644 index 00000000000..a666b11013b --- /dev/null +++ b/db/migrate/20190905223800_add_interruptible_to_builds_metadata.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddInterruptibleToBuildsMetadata < ActiveRecord::Migration[5.0] + DOWNTIME = false + + def change + add_column :ci_builds_metadata, :interruptible, :boolean + end +end diff --git a/db/migrate/20190905223900_add_concurrent_index_to_builds_metadata.rb b/db/migrate/20190905223900_add_concurrent_index_to_builds_metadata.rb new file mode 100644 index 00000000000..d394f995673 --- /dev/null +++ b/db/migrate/20190905223900_add_concurrent_index_to_builds_metadata.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddConcurrentIndexToBuildsMetadata < ActiveRecord::Migration[5.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_builds_metadata, [:build_id], + where: "interruptible = false", + name: "index_ci_builds_metadata_on_build_id_and_interruptible_false" + end + + def down + remove_concurrent_index_by_name(:ci_builds_metadata, 'index_ci_builds_metadata_on_build_id_and_interruptible_false') + end +end diff --git a/db/schema.rb b/db/schema.rb index f2d6f70217b..61f7787f192 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_09_02_160015) do +ActiveRecord::Schema.define(version: 2019_09_05_223900) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -605,6 +605,7 @@ ActiveRecord::Schema.define(version: 2019_09_02_160015) do t.index ["scheduled_at"], name: "partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs", where: "((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text))" t.index ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)" t.index ["stage_id"], name: "index_ci_builds_on_stage_id" + t.index ["status", "created_at", "project_id"], name: "ci_builds_gitlab_monitor_metrics", where: "((type)::text = 'Ci::Build'::text)" t.index ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id" t.index ["token"], name: "index_ci_builds_on_token", unique: true t.index ["token_encrypted"], name: "index_ci_builds_on_token_encrypted", unique: true, where: "(token_encrypted IS NOT NULL)" @@ -618,9 +619,11 @@ ActiveRecord::Schema.define(version: 2019_09_02_160015) do t.integer "project_id", null: false t.integer "timeout" t.integer "timeout_source", default: 1, null: false + t.boolean "interruptible" t.jsonb "config_options" t.jsonb "config_variables" t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true + t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id_and_interruptible_false", where: "(interruptible = false)" t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id" end @@ -2931,6 +2934,8 @@ ActiveRecord::Schema.define(version: 2019_09_02_160015) do t.string "name", null: false t.datetime "created_at" t.datetime "updated_at" + t.boolean "code_owner_approval_required", default: false, null: false + t.index ["project_id", "code_owner_approval_required"], name: "code_owner_approval_required", where: "(code_owner_approval_required = true)" t.index ["project_id"], name: "index_protected_branches_on_project_id" end @@ -3562,6 +3567,7 @@ ActiveRecord::Schema.define(version: 2019_09_02_160015) do t.index ["state"], name: "index_users_on_state" t.index ["state"], name: "index_users_on_state_and_internal", where: "(ghost IS NOT TRUE)" t.index ["state"], name: "index_users_on_state_and_internal_ee", where: "((ghost IS NOT TRUE) AND (bot_type IS NULL))" + t.index ["unconfirmed_email"], name: "index_users_on_unconfirmed_email", where: "(unconfirmed_email IS NOT NULL)" t.index ["username"], name: "index_users_on_username" t.index ["username"], name: "index_users_on_username_trigram", opclass: :gin_trgm_ops, using: :gin end diff --git a/doc/administration/database_load_balancing.md b/doc/administration/database_load_balancing.md index 64eca0b00f6..50ccebffe73 100644 --- a/doc/administration/database_load_balancing.md +++ b/doc/administration/database_load_balancing.md @@ -212,28 +212,25 @@ without it immediately leading to errors being presented to the users. ## Logging -The load balancer logs various messages, such as: +The load balancer logs various events in +[`database_load_balancing.log`](logs.md#database_load_balancinglog-premium-only), such as - When a host is marked as offline - When a host comes back online - When all secondaries are offline +- When a read is retried on a different host due to a query conflict -Each log message contains the tag `[DB-LB]` to make searching/filtering of such -log entries easier. For example: +The log is structured with each entry a JSON object containing at least: -``` -[DB-LB] Host 10.123.2.5 came back online -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Host 10.123.2.6 came back online -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Marking host 10.123.2.7 as offline -[DB-LB] Host 10.123.2.7 came back online -[DB-LB] Host 10.123.2.7 came back online +- An `event` field useful for filtering. +- A human-readable `message` field. +- Some event-specific metadata. For example, `db_host` +- Contextual information that is always logged. For example, `severity` and `time`. + +For example: + +```json +{"severity":"INFO","time":"2019-09-02T12:12:01.728Z","correlation_id":"abcdefg","event":"host_online","message":"Host came back online","db_host":"111.222.333.444","db_port":null,"tag":"rails.database_load_balancing","environment":"production","hostname":"web-example-1","fqdn":"gitlab.example.com","path":null,"params":null} ``` ## Handling Stale Reads diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 407539885a6..7228cc6948e 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -315,7 +315,7 @@ section to resolve the error. Otherwise, the secret is lost and you'll need to [geo-limitations]: ../replication/index.md#current-limitations [planned-failover]: planned_failover.md [setup-geo]: ../replication/index.md#setup-instructions -[updating-geo]: ../replication/updating_the_geo_nodes.md#upgrading-to-gitlab-105 +[updating-geo]: ../replication/version_specific_updates.md#updating-to-gitlab-105 [sec-tfa]: ../../../security/two_factor_authentication.md#disabling-2fa-for-everyone [gitlab-org/omnibus-gitlab#3058]: https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3058 [gitlab-org/gitlab-ee#4284]: https://gitlab.com/gitlab-org/gitlab-ee/issues/4284 diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index 8c27c4dac4f..fda0ebbbeac 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -1,6 +1,26 @@ # Updating the Geo nodes **(PREMIUM ONLY)** -Depending on which version of Geo you are updating to/from, there may be different steps. +Updating Geo nodes involves performing: + +1. [Version-specific update steps](#version-specific-update-steps), depending on the + version being updated to or from. +1. [General update steps](#general-update-steps), for all updates. + +## Version specific update steps + +Depending on which version of Geo you are updating to/from, there may be +different steps. + +- [Updating to GitLab 12.1](version_specific_updates.md#updating-to-gitlab-121) +- [Updating to GitLab 10.8](version_specific_updates.md#updating-to-gitlab-108) +- [Updating to GitLab 10.6](version_specific_updates.md#updating-to-gitlab-106) +- [Updating to GitLab 10.5](version_specific_updates.md#updating-to-gitlab-105) +- [Updating to GitLab 10.3](version_specific_updates.md#updating-to-gitlab-103) +- [Updating to GitLab 10.2](version_specific_updates.md#updating-to-gitlab-102) +- [Updating to GitLab 10.1](version_specific_updates.md#updating-to-gitlab-101) +- [Updating to GitLab 10.0](version_specific_updates.md#updating-to-gitlab-100) +- [Updating from GitLab 9.3 or older](version_specific_updates.md#updating-from-gitlab-93-or-older) +- [Updating to GitLab 9.0](version_specific_updates.md#updating-to-gitlab-90) ## General update steps @@ -31,428 +51,3 @@ everything is working correctly: is received by **secondary** nodes. If you encounter any issues, please consult the [Geo troubleshooting guide](troubleshooting.md). - -## Upgrading to GitLab 12.1 - -By default, GitLab 12.1 will attempt to automatically upgrade the embedded PostgreSQL server to 10.7 from 9.6. Please see [the omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) for the recommended procedure. - -This can be temporarily disabled by running the following before ugprading: - -```sh -sudo touch /etc/gitlab/disable-postgresql-upgrade -``` - -## Upgrading to GitLab 10.8 - -Before 10.8, broadcast messages would not propagate without flushing the cache on the **secondary** nodes. This has been fixed in 10.8, but requires one last cache flush on each **secondary** node: - -```sh -sudo gitlab-rake cache:clear -``` - -## Upgrading to GitLab 10.6 - -In 10.4, we started to recommend that you define a password for database user (`gitlab`). - -We now require this change as we use this password to enable the Foreign Data Wrapper, as a way to optimize -the Geo Tracking Database. We are also improving security by disabling the use of **trust** -authentication method. - -1. **(primary)** Login to your **primary** node and run: - - ```sh - gitlab-ctl pg-password-md5 gitlab - # Enter password: <your_password_here> - # Confirm password: <your_password_here> - # fca0b89a972d69f00eb3ec98a5838484 - ``` - - Copy the generated hash and edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` - postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - - # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. - gitlab_rails['db_password'] = '<your_password_here>' - ``` - - Still in the configuration file, locate and remove the `trust_auth_cidr_address`: - - ```ruby - postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','1.2.3.4/32'] # <- Remove this - ``` - -1. **(primary)** Reconfigure and restart: - - ```sh - sudo gitlab-ctl reconfigure - sudo gitlab-ctl restart - ``` - -1. **(secondary)** Login to all **secondary** nodes and edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` - postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - - # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. - gitlab_rails['db_password'] = '<your_password_here>' - - # Enable Foreign Data Wrapper - geo_secondary['db_fdw'] = true - - # Secondary address in CIDR format, for example '5.6.7.8/32' - postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32'] - ``` - - Still in the configuration file, locate and remove the `trust_auth_cidr_address`: - - ```ruby - postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','5.6.7.8/32'] # <- Remove this - ``` - -1. **(secondary)** Reconfigure and restart: - - ```sh - sudo gitlab-ctl reconfigure - sudo gitlab-ctl restart - ``` - -## Upgrading to GitLab 10.5 - -For Geo Disaster Recovery to work with minimum downtime, your **secondary** node -should use the same set of secrets as the **primary** node. However, setup instructions -prior to the 10.5 release only synchronized the `db_key_base` secret. - -To rectify this error on existing installations, you should **overwrite** the -contents of `/etc/gitlab/gitlab-secrets.json` on each **secondary** node with the -contents of `/etc/gitlab/gitlab-secrets.json` on the **primary** node, then run the -following command on each **secondary** node: - -```sh -sudo gitlab-ctl reconfigure -``` - -If you do not perform this step, you may find that two-factor authentication -[is broken following DR](../disaster_recovery/index.html#i-followed-the-disaster-recovery-instructions-and-now-two-factor-auth-is-broken). - -To prevent SSH requests to the newly promoted **primary** node from failing -due to SSH host key mismatch when updating the **primary** node domain's DNS record -you should perform the step to [Manually replicate **primary** SSH host keys](configuration.md#step-2-manually-replicate-the-primary-nodes-ssh-host-keys) in each -**secondary** node. - -## Upgrading to GitLab 10.4 - -There are no Geo-specific steps to take! - -## Upgrading to GitLab 10.3 - -### Support for SSH repository synchronization removed - -In GitLab 10.2, synchronizing secondaries over SSH was deprecated. In 10.3, -support is removed entirely. All installations will switch to the HTTP/HTTPS -cloning method instead. Before upgrading, ensure that all your Geo nodes are -configured to use this method and that it works for your installation. In -particular, ensure that [Git access over HTTP/HTTPS is enabled](configuration.md#step-6-enable-git-access-over-httphttps). - -Synchronizing repositories over the public Internet using HTTP is insecure, so -you should ensure that you have HTTPS configured before upgrading. Note that -file synchronization is **also** insecure in these cases! - -## Upgrading to GitLab 10.2 - -### Secure PostgreSQL replication - -Support for TLS-secured PostgreSQL replication has been added. If you are -currently using PostgreSQL replication across the open internet without an -external means of securing the connection (e.g., a site-to-site VPN), then you -should immediately reconfigure your **primary** and **secondary** PostgreSQL instances -according to the [updated instructions][database]. - -If you *are* securing the connections externally and wish to continue doing so, -ensure you include the new option `--sslmode=prefer` in future invocations of -`gitlab-ctl replicate-geo-database`. - -### HTTPS repository sync - -Support for replicating repositories and wikis over HTTP/HTTPS has been added. -Replicating over SSH has been deprecated, and support for this option will be -removed in a future release. - -To switch to HTTP/HTTPS replication, log into the **primary** node as an admin and visit -**Admin Area > Geo** (`/admin/geo/nodes`). For each **secondary** node listed, -press the "Edit" button, change the "Repository cloning" setting from -"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take -effect immediately. - -Any new secondaries should be created using HTTP/HTTPS replication - this is the -default setting. - -After you've verified that HTTP/HTTPS replication is working, you should remove -the now-unused SSH keys from your secondaries, as they may cause problems if the -**secondary** node if ever promoted to a **primary** node: - -1. **(secondary)** Login to **all** your **secondary** nodes and run: - - ```ruby - sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub - ``` - -### Hashed Storage - -CAUTION: **Warning:** -Hashed storage is in **Alpha**. It is considered experimental and not -production-ready. See [Hashed Storage] for more detail. - -If you previously enabled Hashed Storage and migrated all your existing -projects to Hashed Storage, disabling hashed storage will not migrate projects -to their previous project based storage path. As such, once enabled and -migrated we recommend leaving Hashed Storage enabled. - -## Upgrading to GitLab 10.1 - -CAUTION: **Warning:** -Hashed storage is in **Alpha**. It is considered experimental and not -production-ready. See [Hashed Storage] for more detail. - -[Hashed storage] was introduced in GitLab 10.0, and a [migration path][hashed-migration] -for existing repositories was added in GitLab 10.1. - -## Upgrading to GitLab 10.0 - -Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via -the database][ssh-fast-lookup] to avoid having to maintain consistency of the -`authorized_keys` file for SSH access. Failing to do this will prevent users -from being able to clone via SSH. - -Note that in older versions of Geo, attachments downloaded on the **secondary** -nodes would be saved to the wrong directory. We recommend that you do the -following to clean this up. - -On the **secondary** Geo nodes, run as root: - -```sh -mv /var/opt/gitlab/gitlab-rails/working /var/opt/gitlab/gitlab-rails/working.old -mkdir /var/opt/gitlab/gitlab-rails/working -chmod 700 /var/opt/gitlab/gitlab-rails/working -chown git:git /var/opt/gitlab/gitlab-rails/working -``` - -You may delete `/var/opt/gitlab/gitlab-rails/working.old` any time. - -Once this is done, we advise restarting GitLab on the **secondary** nodes for the -new working directory to be used: - -```sh -sudo gitlab-ctl restart -``` - -## Upgrading from GitLab 9.3 or older - -If you started running Geo on GitLab 9.3 or older, we recommend that you -resync your **secondary** PostgreSQL databases to use replication slots. If you -started using Geo with GitLab 9.4 or 10.x, no further action should be -required because replication slots are used by default. However, if you -started with GitLab 9.3 and upgraded later, you should still follow the -instructions below. - -When in doubt, it does not hurt to do a resync. The easiest way to do this in -Omnibus is the following: - -1. Make sure you have Omnibus GitLab on the **primary** server. -1. Run `gitlab-ctl reconfigure` and `gitlab-ctl restart postgresql`. This will enable replication slots on the **primary** database. -1. Check the steps about defining `postgresql['sql_user_password']`, `gitlab_rails['db_password']`. -1. Make sure `postgresql['max_replication_slots']` matches the number of **secondary** Geo nodes locations. -1. Install GitLab on the **secondary** server. -1. Re-run the [database replication process](database.md#step-3-initiate-the-replication-process). - -## Special update notes for 9.0.x - -> **IMPORTANT**: -With GitLab 9.0, the PostgreSQL version is upgraded to 9.6 and manual steps are -required in order to update the **secondary** nodes and keep the Streaming -Replication working. Downtime is required, so plan ahead. - -The following steps apply only if you upgrade from a 8.17 GitLab version to -9.0+. For previous versions, update to GitLab 8.17 first before attempting to -upgrade to 9.0+. - ---- - -Make sure to follow the steps in the exact order as they appear below and pay -extra attention in what node (either **primary** or **secondary**) you execute them! Each step -is prepended with the relevant node for better clarity: - -1. **(secondary)** Login to **all** your **secondary** nodes and stop all services: - - ```ruby - sudo gitlab-ctl stop - ``` - -1. **(secondary)** Make a backup of the `recovery.conf` file on **all** - **secondary** nodes to preserve PostgreSQL's credentials: - - ```sh - sudo cp /var/opt/gitlab/postgresql/data/recovery.conf /var/opt/gitlab/ - ``` - -1. **(primary)** Update the **primary** node to GitLab 9.0 following the - [regular update docs][update]. At the end of the update, the **primary** node - will be running with PostgreSQL 9.6. - -1. **(primary)** To prevent a de-synchronization of the repository replication, - stop all services except `postgresql` as we will use it to re-initialize the - **secondary** node's database: - - ```sh - sudo gitlab-ctl stop - sudo gitlab-ctl start postgresql - ``` - -1. **(secondary)** Run the following steps on each of the **secondary** nodes: - - 1. **(secondary)** Stop all services: - - ```sh - sudo gitlab-ctl stop - ``` - - 1. **(secondary)** Prevent running database migrations: - - ```sh - sudo touch /etc/gitlab/skip-auto-migrations - ``` - - 1. **(secondary)** Move the old database to another directory: - - ```sh - sudo mv /var/opt/gitlab/postgresql{,.bak} - ``` - - 1. **(secondary)** Update to GitLab 9.0 following the [regular update docs][update]. - At the end of the update, the node will be running with PostgreSQL 9.6. - - 1. **(secondary)** Make sure all services are up: - - ```sh - sudo gitlab-ctl start - ``` - - 1. **(secondary)** Reconfigure GitLab: - - ```sh - sudo gitlab-ctl reconfigure - ``` - - 1. **(secondary)** Run the PostgreSQL upgrade command: - - ```sh - sudo gitlab-ctl pg-upgrade - ``` - - 1. **(secondary)** See the stored credentials for the database that you will - need to re-initialize the replication: - - ```sh - sudo grep -s primary_conninfo /var/opt/gitlab/recovery.conf - ``` - - 1. **(secondary)** Save the snippet below in a file, let's say `/tmp/replica.sh`. Modify the - embedded paths if necessary: - - ``` - #!/bin/bash - - PORT="5432" - USER="gitlab_replicator" - echo --------------------------------------------------------------- - echo WARNING: Make sure this script is run from the secondary server - echo --------------------------------------------------------------- - echo - echo Enter the IP or FQDN of the primary PostgreSQL server - read HOST - echo Enter the password for $USER@$HOST - read -s PASSWORD - echo Enter the required sslmode - read SSLMODE - - echo Stopping PostgreSQL and all GitLab services - sudo service gitlab stop - sudo service postgresql stop - - echo Backing up postgresql.conf - sudo -u postgres mv /var/opt/gitlab/postgresql/data/postgresql.conf /var/opt/gitlab/postgresql/ - - echo Cleaning up old cluster directory - sudo -u postgres rm -rf /var/opt/gitlab/postgresql/data - - echo Starting base backup as the replicator user - echo Enter the password for $USER@$HOST - sudo -u postgres /opt/gitlab/embedded/bin/pg_basebackup -h $HOST -D /var/opt/gitlab/postgresql/data -U gitlab_replicator -v -x -P - - echo Writing recovery.conf file - sudo -u postgres bash -c "cat > /var/opt/gitlab/postgresql/data/recovery.conf <<- _EOF1_ - standby_mode = 'on' - primary_conninfo = 'host=$HOST port=$PORT user=$USER password=$PASSWORD sslmode=$SSLMODE' - _EOF1_ - " - - echo Restoring postgresql.conf - sudo -u postgres mv /var/opt/gitlab/postgresql/postgresql.conf /var/opt/gitlab/postgresql/data/ - - echo Starting PostgreSQL - sudo service postgresql start - ``` - - 1. **(secondary)** Run the recovery script using the credentials from the - previous step: - - ```sh - sudo bash /tmp/replica.sh - ``` - - 1. **(secondary)** Reconfigure GitLab: - - ```sh - sudo gitlab-ctl reconfigure - ``` - - 1. **(secondary)** Start all services: - - ```sh - sudo gitlab-ctl start - ``` - - 1. **(secondary)** Repeat the steps for the remaining **secondary** nodes. - -1. **(primary)** After all **secondary** nodes are updated, start all services in - **primary** node: - - ```sh - sudo gitlab-ctl start - ``` - -### Update tracking database on **secondary** node - -After updating a **secondary** node, you might need to run migrations on -the tracking database. The tracking database was added in GitLab 9.1, -and it is required since 10.0. - -1. Run database migrations on tracking database: - - ```sh - sudo gitlab-rake geo:db:migrate - ``` - -1. Repeat this step for each **secondary** node. - -[update]: ../../../update/README.md -[database]: database.md -[Hashed Storage]: ../../repository_storage_types.md -[hashed-migration]: ../../raketasks/storage.md -[ssh-fast-lookup]: ../../operations/fast_ssh_key_lookup.md diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md new file mode 100644 index 00000000000..6d550a49df4 --- /dev/null +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -0,0 +1,426 @@ +# Version specific update instructions + +Check this document if it includes instructions for the version you are updating. +These steps go together with the [general steps](updating_the_geo_nodes.md#general-update-steps) +for updating Geo nodes. + +## Updating to GitLab 12.1 + +By default, GitLab 12.1 will attempt to automatically update the +embedded PostgreSQL server to 10.7 from 9.6. Please see +[the omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```sh +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 10.8 + +Before 10.8, broadcast messages would not propagate without flushing +the cache on the **secondary** nodes. This has been fixed in 10.8, but +requires one last cache flush on each **secondary** node: + +```sh +sudo gitlab-rake cache:clear +``` + +## Updating to GitLab 10.6 + +In 10.4, we started to recommend that you define a password for database user (`gitlab`). + +We now require this change as we use this password to enable the Foreign Data Wrapper, as a way to optimize +the Geo Tracking Database. We are also improving security by disabling the use of **trust** +authentication method. + +1. **(primary)** Login to your **primary** node and run: + + ```sh + gitlab-ctl pg-password-md5 gitlab + # Enter password: <your_password_here> + # Confirm password: <your_password_here> + # fca0b89a972d69f00eb3ec98a5838484 + ``` + + Copy the generated hash and edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + + # Every node that runs Unicorn or Sidekiq needs to have the database + # password specified as below. If you have a high-availability setup, this + # must be present in all application nodes. + gitlab_rails['db_password'] = '<your_password_here>' + ``` + + Still in the configuration file, locate and remove the `trust_auth_cidr_address`: + + ```ruby + postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','1.2.3.4/32'] # <- Remove this + ``` + +1. **(primary)** Reconfigure and restart: + + ```sh + sudo gitlab-ctl reconfigure + sudo gitlab-ctl restart + ``` + +1. **(secondary)** Login to all **secondary** nodes and edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + + # Every node that runs Unicorn or Sidekiq needs to have the database + # password specified as below. If you have a high-availability setup, this + # must be present in all application nodes. + gitlab_rails['db_password'] = '<your_password_here>' + + # Enable Foreign Data Wrapper + geo_secondary['db_fdw'] = true + + # Secondary address in CIDR format, for example '5.6.7.8/32' + postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32'] + ``` + + Still in the configuration file, locate and remove the `trust_auth_cidr_address`: + + ```ruby + postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','5.6.7.8/32'] # <- Remove this + ``` + +1. **(secondary)** Reconfigure and restart: + + ```sh + sudo gitlab-ctl reconfigure + sudo gitlab-ctl restart + ``` + +## Updating to GitLab 10.5 + +For Geo Disaster Recovery to work with minimum downtime, your **secondary** node +should use the same set of secrets as the **primary** node. However, setup instructions +prior to the 10.5 release only synchronized the `db_key_base` secret. + +To rectify this error on existing installations, you should **overwrite** the +contents of `/etc/gitlab/gitlab-secrets.json` on each **secondary** node with the +contents of `/etc/gitlab/gitlab-secrets.json` on the **primary** node, then run the +following command on each **secondary** node: + +```sh +sudo gitlab-ctl reconfigure +``` + +If you do not perform this step, you may find that two-factor authentication +[is broken following DR](../disaster_recovery/index.html#i-followed-the-disaster-recovery-instructions-and-now-two-factor-auth-is-broken). + +To prevent SSH requests to the newly promoted **primary** node from failing +due to SSH host key mismatch when updating the **primary** node domain's DNS record +you should perform the step to [Manually replicate **primary** SSH host keys](configuration.md#step-2-manually-replicate-the-primary-nodes-ssh-host-keys) in each +**secondary** node. + +## Updating to GitLab 10.3 + +### Support for SSH repository synchronization removed + +In GitLab 10.2, synchronizing secondaries over SSH was deprecated. In 10.3, +support is removed entirely. All installations will switch to the HTTP/HTTPS +cloning method instead. Before updating, ensure that all your Geo nodes are +configured to use this method and that it works for your installation. In +particular, ensure that [Git access over HTTP/HTTPS is enabled](configuration.md#step-6-enable-git-access-over-httphttps). + +Synchronizing repositories over the public Internet using HTTP is insecure, so +you should ensure that you have HTTPS configured before updating. Note that +file synchronization is **also** insecure in these cases! + +## Updating to GitLab 10.2 + +### Secure PostgreSQL replication + +Support for TLS-secured PostgreSQL replication has been added. If you are +currently using PostgreSQL replication across the open internet without an +external means of securing the connection (e.g., a site-to-site VPN), then you +should immediately reconfigure your **primary** and **secondary** PostgreSQL instances +according to the [updated instructions](database.md). + +If you *are* securing the connections externally and wish to continue doing so, +ensure you include the new option `--sslmode=prefer` in future invocations of +`gitlab-ctl replicate-geo-database`. + +### HTTPS repository sync + +Support for replicating repositories and wikis over HTTP/HTTPS has been added. +Replicating over SSH has been deprecated, and support for this option will be +removed in a future release. + +To switch to HTTP/HTTPS replication, log into the **primary** node as an admin and visit +**Admin Area > Geo** (`/admin/geo/nodes`). For each **secondary** node listed, +press the "Edit" button, change the "Repository cloning" setting from +"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take +effect immediately. + +Any new secondaries should be created using HTTP/HTTPS replication - this is the +default setting. + +After you've verified that HTTP/HTTPS replication is working, you should remove +the now-unused SSH keys from your secondaries, as they may cause problems if the +**secondary** node if ever promoted to a **primary** node: + +1. **(secondary)** Login to **all** your **secondary** nodes and run: + + ```ruby + sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub + ``` + +### Hashed Storage + +CAUTION: **Warning:** +Hashed storage is in **Alpha**. It is considered experimental and not +production-ready. See [Hashed Storage](../../repository_storage_types.md) for more detail. + +If you previously enabled Hashed Storage and migrated all your existing +projects to Hashed Storage, disabling hashed storage will not migrate projects +to their previous project based storage path. As such, once enabled and +migrated we recommend leaving Hashed Storage enabled. + +## Updating to GitLab 10.1 + +CAUTION: **Warning:** +Hashed storage is in **Alpha**. It is considered experimental and not +production-ready. See [Hashed Storage](../../repository_storage_types.md) for more detail. + +[Hashed storage](../../repository_storage_types.md) was introduced in +GitLab 10.0, and a [migration path](../../raketasks/storage.md) for +existing repositories was added in GitLab 10.1. + +## Updating to GitLab 10.0 + +Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via +the database](../../operations/fast_ssh_key_lookup.md) to avoid having to maintain consistency of the +`authorized_keys` file for SSH access. Failing to do this will prevent users +from being able to clone via SSH. + +Note that in older versions of Geo, attachments downloaded on the **secondary** +nodes would be saved to the wrong directory. We recommend that you do the +following to clean this up. + +On the **secondary** Geo nodes, run as root: + +```sh +mv /var/opt/gitlab/gitlab-rails/working /var/opt/gitlab/gitlab-rails/working.old +mkdir /var/opt/gitlab/gitlab-rails/working +chmod 700 /var/opt/gitlab/gitlab-rails/working +chown git:git /var/opt/gitlab/gitlab-rails/working +``` + +You may delete `/var/opt/gitlab/gitlab-rails/working.old` any time. + +Once this is done, we advise restarting GitLab on the **secondary** nodes for the +new working directory to be used: + +```sh +sudo gitlab-ctl restart +``` + +## Updating from GitLab 9.3 or older + +If you started running Geo on GitLab 9.3 or older, we recommend that you +resync your **secondary** PostgreSQL databases to use replication slots. If you +started using Geo with GitLab 9.4 or 10.x, no further action should be +required because replication slots are used by default. However, if you +started with GitLab 9.3 and updated later, you should still follow the +instructions below. + +When in doubt, it does not hurt to do a resync. The easiest way to do this in +Omnibus is the following: + +1. Make sure you have Omnibus GitLab on the **primary** server. +1. Run `gitlab-ctl reconfigure` and `gitlab-ctl restart postgresql`. This will enable replication slots on the **primary** database. +1. Check the steps about defining `postgresql['sql_user_password']`, `gitlab_rails['db_password']`. +1. Make sure `postgresql['max_replication_slots']` matches the number of **secondary** Geo nodes locations. +1. Install GitLab on the **secondary** server. +1. Re-run the [database replication process](database.md#step-3-initiate-the-replication-process). + +## Updating to GitLab 9.0 + +> **IMPORTANT**: +With GitLab 9.0, the PostgreSQL version is updated to 9.6 and manual steps are +required in order to update the **secondary** nodes and keep the Streaming +Replication working. Downtime is required, so plan ahead. + +The following steps apply only if you update from a 8.17 GitLab version to +9.0+. For previous versions, update to GitLab 8.17 first before attempting to +update to 9.0+. + +--- + +Make sure to follow the steps in the exact order as they appear below and pay +extra attention in what node (either **primary** or **secondary**) you execute them! Each step +is prepended with the relevant node for better clarity: + +1. **(secondary)** Log in to **all** your **secondary** nodes and stop all services: + + ```ruby + sudo gitlab-ctl stop + ``` + +1. **(secondary)** Make a backup of the `recovery.conf` file on **all** + **secondary** nodes to preserve PostgreSQL's credentials: + + ```sh + sudo cp /var/opt/gitlab/postgresql/data/recovery.conf /var/opt/gitlab/ + ``` + +1. **(primary)** Update the **primary** node to GitLab 9.0 following the + [regular update docs](../../../update/README.md). At the end of the + update, the **primary** node will be running with PostgreSQL 9.6. + +1. **(primary)** To prevent a de-synchronization of the repository replication, + stop all services except `postgresql` as we will use it to re-initialize the + **secondary** node's database: + + ```sh + sudo gitlab-ctl stop + sudo gitlab-ctl start postgresql + ``` + +1. **(secondary)** Run the following steps on each of the **secondary** nodes: + + 1. **(secondary)** Stop all services: + + ```sh + sudo gitlab-ctl stop + ``` + + 1. **(secondary)** Prevent running database migrations: + + ```sh + sudo touch /etc/gitlab/skip-auto-migrations + ``` + + 1. **(secondary)** Move the old database to another directory: + + ```sh + sudo mv /var/opt/gitlab/postgresql{,.bak} + ``` + + 1. **(secondary)** Update to GitLab 9.0 following the [regular update docs](../../../update/README.md). + At the end of the update, the node will be running with PostgreSQL 9.6. + + 1. **(secondary)** Make sure all services are up: + + ```sh + sudo gitlab-ctl start + ``` + + 1. **(secondary)** Reconfigure GitLab: + + ```sh + sudo gitlab-ctl reconfigure + ``` + + 1. **(secondary)** Run the PostgreSQL upgrade command: + + ```sh + sudo gitlab-ctl pg-upgrade + ``` + + 1. **(secondary)** See the stored credentials for the database that you will + need to re-initialize the replication: + + ```sh + sudo grep -s primary_conninfo /var/opt/gitlab/recovery.conf + ``` + + 1. **(secondary)** Save the snippet below in a file, let's say `/tmp/replica.sh`. Modify the + embedded paths if necessary: + + ``` + #!/bin/bash + + PORT="5432" + USER="gitlab_replicator" + echo --------------------------------------------------------------- + echo WARNING: Make sure this script is run from the secondary server + echo --------------------------------------------------------------- + echo + echo Enter the IP or FQDN of the primary PostgreSQL server + read HOST + echo Enter the password for $USER@$HOST + read -s PASSWORD + echo Enter the required sslmode + read SSLMODE + + echo Stopping PostgreSQL and all GitLab services + sudo service gitlab stop + sudo service postgresql stop + + echo Backing up postgresql.conf + sudo -u postgres mv /var/opt/gitlab/postgresql/data/postgresql.conf /var/opt/gitlab/postgresql/ + + echo Cleaning up old cluster directory + sudo -u postgres rm -rf /var/opt/gitlab/postgresql/data + + echo Starting base backup as the replicator user + echo Enter the password for $USER@$HOST + sudo -u postgres /opt/gitlab/embedded/bin/pg_basebackup -h $HOST -D /var/opt/gitlab/postgresql/data -U gitlab_replicator -v -x -P + + echo Writing recovery.conf file + sudo -u postgres bash -c "cat > /var/opt/gitlab/postgresql/data/recovery.conf <<- _EOF1_ + standby_mode = 'on' + primary_conninfo = 'host=$HOST port=$PORT user=$USER password=$PASSWORD sslmode=$SSLMODE' + _EOF1_ + " + + echo Restoring postgresql.conf + sudo -u postgres mv /var/opt/gitlab/postgresql/postgresql.conf /var/opt/gitlab/postgresql/data/ + + echo Starting PostgreSQL + sudo service postgresql start + ``` + + 1. **(secondary)** Run the recovery script using the credentials from the + previous step: + + ```sh + sudo bash /tmp/replica.sh + ``` + + 1. **(secondary)** Reconfigure GitLab: + + ```sh + sudo gitlab-ctl reconfigure + ``` + + 1. **(secondary)** Start all services: + + ```sh + sudo gitlab-ctl start + ``` + + 1. **(secondary)** Repeat the steps for the remaining **secondary** nodes. + +1. **(primary)** After all **secondary** nodes are updated, start all services in + **primary** node: + + ```sh + sudo gitlab-ctl start + ``` + +### Update tracking database on **secondary** node + +After updating a **secondary** node, you might need to run migrations on +the tracking database. The tracking database was added in GitLab 9.1, +and it is required since 10.0. + +1. Run database migrations on tracking database: + + ```sh + sudo gitlab-rake geo:db:migrate + ``` + +1. Repeat this step for each **secondary** node. diff --git a/doc/administration/index.md b/doc/administration/index.md index 711eebcd61e..d557068e6c8 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -187,13 +187,29 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Debugging tips](troubleshooting/debug.md): Tips to debug problems when things go wrong - [Log system](logs.md): Where to look for logs. - [Sidekiq Troubleshooting](troubleshooting/sidekiq.md): Debug when Sidekiq appears hung and is not processing jobs. -- Useful [diagnostics tools](troubleshooting/diagnostics_tools.md) that are sometimes used by the GitLab - Support team. -- [Troubleshooting ElasticSearch](troubleshooting/elasticsearch.md): Tips to troubleshoot ElasticSearch. -- [Kubernetes troubleshooting](troubleshooting/kubernetes_cheat_sheet.md): Commands and tips useful - for troubleshooting Kubernetes-related issues. -- Useful links from the Support Team: - - [GitLab Developer Docs](https://docs.gitlab.com/ee/development/README.html). - - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html). - - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html). - - [Strace zine](https://wizardzines.com/zines/strace/). +- [Troubleshooting ElasticSearch](troubleshooting/elasticsearch.md) + +### Support Team Docs + +The GitLab Support Team has collected a lot of information about troubleshooting GitLab +instances. These documents are normally used by the Support Team itself, or by customers +with direct guidance from a Support Team member. GitLab administrators may find the +information useful for troubleshooting, but if you are experiencing trouble with your +GitLab instance, you should check your [support options](https://about.gitlab.com/support/) +before referring to these documents. + +CAUTION: **Warning:** +Using the commands listed in the documentation below could result in data loss or +other damage to a GitLab instance, and should only be used by experienced administrators +who are aware of the risks. + +- [Useful diagnostics tools](troubleshooting/diagnostics_tools.md) +- [Useful Linux commands](troubleshooting/linux_cheat_sheet.md) +- [Troubleshooting Kubernetes](troubleshooting/kubernetes_cheat_sheet.md) +- [Guide to test environments](troubleshooting/test_environments.md) (for Support Engineers) +- [GitLab rails console commands](troubleshooting/gitlab_rails_cheat_sheet.md) (for Support Engineers) +- Useful links: + - [GitLab Developer Docs](../development/README.md) + - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html) + - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html) + - [Strace zine](https://wizardzines.com/zines/strace/) diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 4c43a434817..c51b53c596e 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -339,11 +339,11 @@ installations from source. [Rack Attack]: ../security/rack_attack.md [Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md -## `database_load_balancing.log` +## `database_load_balancing.log` **(PREMIUM ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/15442) in GitLab 12.3. -This log is used for observability of [Database Load Balancing](database_load_balancing.md). +Contains details of GitLab's [Database Load Balancing](database_load_balancing.md). It is stored at: - `/var/log/gitlab/gitlab-rails/database_load_balancing.log` for Omnibus GitLab packages. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index ca48326c6d0..6dbfd5404d0 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -91,33 +91,6 @@ The following metrics can be controlled by feature flags: |:---------------------------------------------------------------|:-------------------------------------------------------------------| | `gitlab_method_call_duration_seconds` | `prometheus_metrics_method_instrumentation` | | `gitlab_transaction_allocated_memory_bytes` | `prometheus_metrics_transaction_allocated_memory` | -| `gitlab_transaction_event_build_found_total` | `prometheus_transaction_event_build_found_total` | -| `gitlab_transaction_event_build_invalid_total` | `prometheus_transaction_event_build_invalid_total` | -| `gitlab_transaction_event_build_not_found_cached_total` | `prometheus_transaction_event_build_not_found_cached_total` | -| `gitlab_transaction_event_build_not_found_total` | `prometheus_transaction_event_build_not_found_total` | -| `gitlab_transaction_event_change_default_branch_total` | `prometheus_transaction_event_change_default_branch_total` | -| `gitlab_transaction_event_create_repository_total` | `prometheus_transaction_event_create_repository_total` | -| `gitlab_transaction_event_etag_caching_cache_hit_total` | `prometheus_transaction_event_etag_caching_cache_hit_total` | -| `gitlab_transaction_event_etag_caching_header_missing_total` | `prometheus_transaction_event_etag_caching_header_missing_total` | -| `gitlab_transaction_event_etag_caching_key_not_found_total` | `prometheus_transaction_event_etag_caching_key_not_found_total` | -| `gitlab_transaction_event_etag_caching_middleware_used_total` | `prometheus_transaction_event_etag_caching_middleware_used_total` | -| `gitlab_transaction_event_etag_caching_resource_changed_total` | `prometheus_transaction_event_etag_caching_resource_changed_total` | -| `gitlab_transaction_event_fork_repository_total` | `prometheus_transaction_event_fork_repository_total` | -| `gitlab_transaction_event_import_repository_total` | `prometheus_transaction_event_import_repository_total` | -| `gitlab_transaction_event_push_branch_total` | `prometheus_transaction_event_push_branch_total` | -| `gitlab_transaction_event_push_commit_total` | `prometheus_transaction_event_push_commit_total` | -| `gitlab_transaction_event_push_tag_total` | `prometheus_transaction_event_push_tag_total` | -| `gitlab_transaction_event_rails_exception_total` | `prometheus_transaction_event_rails_exception_total` | -| `gitlab_transaction_event_receive_email_total` | `prometheus_transaction_event_receive_email_total` | -| `gitlab_transaction_event_remote_mirrors_failed_total` | `prometheus_transaction_event_remote_mirrors_failed_total` | -| `gitlab_transaction_event_remote_mirrors_finished_total` | `prometheus_transaction_event_remote_mirrors_finished_total` | -| `gitlab_transaction_event_remote_mirrors_running_total` | `prometheus_transaction_event_remote_mirrors_running_total` | -| `gitlab_transaction_event_remove_branch_total` | `prometheus_transaction_event_remove_branch_total` | -| `gitlab_transaction_event_remove_repository_total` | `prometheus_transaction_event_remove_repository_total` | -| `gitlab_transaction_event_remove_tag_total` | `prometheus_transaction_event_remove_tag_total` | -| `gitlab_transaction_event_sidekiq_exception_total` | `prometheus_transaction_event_sidekiq_exception_total` | -| `gitlab_transaction_event_stuck_import_jobs_total` | `prometheus_transaction_event_stuck_import_jobs_total` | -| `gitlab_transaction_event_update_build_total` | `prometheus_transaction_event_update_build_total` | | `gitlab_view_rendering_duration_seconds` | `prometheus_metrics_view_instrumentation` | ## Sidekiq Metrics available for Geo **(PREMIUM)** diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md new file mode 100644 index 00000000000..0c5611aa6cd --- /dev/null +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -0,0 +1,1040 @@ +--- +type: reference +--- + +# GitLab Rails Console Cheat Sheet + +This is the GitLab Support Team's collection of information regarding the GitLab rails +console, for use while troubleshooting. It is listed here for transparency, +and it may be useful for users with experience with these tools. If you are currently +having an issue with GitLab, it is highly recommended that you check your +[support options](https://about.gitlab.com/support/) first, before attempting to use +this information. + +CAUTION: **CAUTION:** +Please note that some of these scripts could be damaging if not run correctly, +or under the right conditions. We highly recommend running them under the +guidance of a Support Engineer, or running them in a test environment with a +backup of the instance ready to be restored, just in case. + +CAUTION: **CAUTION:** +Please also note that as GitLab changes, changes to the code are inevitable, +and so some scripts may not work as they once used to. These are not kept +up-to-date as these scripts/commands were added as they were found/needed. As +mentioned above, we recommend running these scripts under the supervision of a +Support Engineer, who can also verify that they will continue to work as they +should and, if needed, update the script for the latest version of GitLab. + +## Use the Rails Runner + +If the script you want to run is short, you can use the Rails Runner to avoid +entering the rails console in the first place. Here's an example of its use: + +```bash +gitlab-rails runner "RAILS_COMMAND" + +# Example with a 2-line script +gitlab-rails runner "user = User.first; puts user.username" +``` + +## Enable debug logging on rails console + +```ruby +Rails.logger.level = 0 +``` + +## Enable debug logging for ActiveRecord (db issues) + +```ruby +ActiveRecord::Base.logger = Logger.new(STDOUT) +``` + +## Temporarily Disable Timeout + +```ruby +ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') +``` + +## Find specific methods for an object + +```ruby +Array.methods.select { |m| m.to_s.include? "sing" } +Array.methods.grep(/sing/) +``` + +## Find method source + +Works for [non-instrumented methods](https://docs.gitlab.com/ce/development/instrumentation.html#checking-instrumented-methods): + +```ruby +instance_of_object.method(:foo).source_location + +# Example for when we would call project.private? +project.method(:private?).source_location +``` + +## Query an object + +```ruby +o = Object.where('attribute like ?', 'ex') +``` + +## View all keys in cache + +```ruby +Rails.cache.instance_variable_get(:@data).keys +``` + +## Rails console history + +```ruby +puts Readline::HISTORY.to_a +``` + +## Profile a page + +```ruby +# Before 11.6.0 +logger = Logger.new(STDOUT) +admin_token = User.find_by_username('ADMIN_USERNAME').personal_access_tokens.first.token +app.get("URL/?private_token=#{admin_token}") + +# From 11.6.0 +admin = User.find_by_username('ADMIN_USERNAME') +url = "/url/goes/here" +Gitlab::Profiler.with_user(admin) { app.get(url) } +``` + +## Using the GitLab profiler inside console (used as of 10.5) + +```ruby +logger = Logger.new(STDOUT) +admin = User.find_by_username('ADMIN_USERNAME') +Gitlab::Profiler.profile('URL', logger: logger, user: admin) +``` + +## Time an operation + +```ruby +# A single operation +Benchmark.measure { <operation> } + +# A breakdown of multiple operations +Benchmark.bm do |x| + x.report(:label1) { <operation_1> } + x.report(:label2) { <operation_2> } +end +``` + +## Command Line + +### Check the GitLab version fast + +```bash +grep -m 1 gitlab /opt/gitlab/version-manifest.txt +``` + +### Debugging SSH + +```bash +GIT_SSH_COMMAND="ssh -vvv" git clone <repository> +``` + +### Debugging over HTTPS + +```bash +GIT_CURL_VERBOSE=1 GIT_TRACE=1 git clone <repository> +``` + +## Projects + +### Find projects + +```ruby +# A single project +project = Project.find_by_full_path('PROJECT_PATH') + +# All projects in a particular namespace. Can be a username, a group +# ('gitlab-org'), or even include subgroups ('gitlab-org/distribution') +namespace = Namespace.find_by_full_path('NAMESPACE_PATH') +projects = namespace.all_projects +``` + +### Clear a project's cache + +```ruby +ProjectCacheWorker.perform_async(project.id) +``` + +### Expire the .exists? cache + +```ruby +project.repository.expire_exists_cache +``` + +### Make all projects private + +```ruby +Project.update_all(visibility_level: 0) +``` + +### Find & remove projects that are pending deletion + +```ruby +# +# This section will list all the projects which are pending deletion +# +projects = Project.where(pending_delete: true) +projects.each do |p| + puts "Project name: #{p.id}" + puts "Project name: #{p.name}" + puts "Repository path: #{p.repository.storage_path}" +end + +# +# Assign a user (the root user will do) +# +user = User.find_by_username('root') + +# +# For each project listed repeat these two commands +# + +# Find the project, update the xxx-changeme values from above +project = Project.find_by_full_path('group-changeme/project-changeme') + +# Delete the project +::Projects::DestroyService.new(project, user, {}).execute +``` + +Next, run `sudo gitlab-rake gitlab:cleanup:repos` on the command line to finish. + +### Destroy a project + +```ruby +project = Project.find_by_full_path('') +user = User.find_by_username('') +ProjectDestroyWorker.perform_async(project.id, user.id, {}) +# or ProjectDestroyWorker.new.perform(project.id, user.id, {}) +# or Projects::DestroyService.new(project, user).execute +``` + +### Remove fork relationship manually + +```ruby +p = Project.find_by_full_path('') +u = User.find_by_username('') +::Projects::UnlinkForkService.new(p, u).execute +``` + +### Make a project read-only (can only be done in the console) + +```ruby +# Make a project read-only +project.repository_read_only = true; project.save + +# OR +project.update!(repository_read_only: true) +``` + +### Bulk update service integration password for _all_ projects + +For example, change the Jira user's password for all projects that have the Jira +integration active: + +```ruby +p = Project.find_by_sql("SELECT p.id FROM projects p LEFT JOIN services s ON p.id = s.project_id WHERE s.type = 'JiraService' AND s.active = true") + +p.each do |project| + project.jira_service.update_attribute(:password, '<your-new-password>') +end +``` + +### Identify un-indexed projects + +```ruby +Project.find_each do |project| + puts "id #{project.id}: #{project.namespace.name.to_s}/#{project.name.to_s}" if project.index_status.nil? +end +``` + +## Imports / Exports + +```ruby +# Find the project and get the error +p = Project.find_by_full_path('<username-or-group>/<project-name>') + +p.import_error + +# To finish the import on GitLab running version before 11.6 +p.import_finish + +# To finish the import on GitLab running version 11.6 or after +p.import_state.mark_as_failed("Failed manually through console.") +``` + +### Rename imported repository + +In a specific situation, an imported repository needed to be renamed. The Support +Team was informed of a backup restore that failed on a single repository, which created +the project with an empty repository. The project was successfully restored to a dev +instance, then exported, and imported into a new project under a different name. + +The Support Team was able to transfer the incorrectly named imported project into the +correctly named empty project using the steps below. + +Move the new repository to the empty repository: + +```bash +mv /var/opt/gitlab/git-data/repositories/<group>/<new-project> /var/opt/gitlab/git-data/repositories/<group>/<empty-project> +``` + +Make sure the permissions are correct: + +```bash +chown -R git:git <path-to-directory>.git +``` + +Clear the cache: + +```bash +sudo gitlab-rake cache:clear +``` + +## Repository + +### Search sequence of pushes to a repository + +If it seems that a commit has gone "missing", search the sequence of pushes to a repository. +[This StackOverflow article](https://stackoverflow.com/questions/13468027/the-mystery-of-the-missing-commit-across-merges) +describes how you can end up in this state without a force push. + +If you look at the output from the sample code below for the target branch, you will +see a discontinuity in the from/to commits as you step through the output. Each new +push should be "from" the "to" SHA of the previous push. When this discontinuity happens, +you will see two pushes with the same "from" SHA: + +```ruby +p = Project.find_with_namespace('u/p') +p.events.code_push.last(100).each do |e| + printf "%-20.20s %8s...%8s (%s)\n", e.data[:ref], e.data[:before], e.data[:after], e.author.try(:username) +end +``` + +GitLab 9.5 and above: + +```ruby +p = Project.find_by_full_path('u/p') +p.events.code_push.last(100).each do |e| + printf "%-20.20s %8s...%8s (%s)\n", e.push_event_payload[:ref], e.push_event_payload[:commit_from], e.push_event_payload[:commit_to], e.author.try(:username) +end +``` + +## Mirrors + +### Find mirrors with "bad decrypt" errors + +```ruby +total = 0 +bad = [] +ProjectImportData.find_each do |data| + begin + total += 1 + data.credentials + rescue => e + bad << data + end +end + +puts "Bad count: #{bad.count} / #{total}" +bad.each do |repo| + puts Project.find(repo.project_id).full_path +end; bad.count +``` + +### Transfer mirror users and tokens to a single service account + +Use case: If you have multiple users using their own GitHub credentials to set up +repository mirroring, mirroring breaks when people leave the company. Use this +script to migrate disparate mirroring users and tokens into a single service account: + +```ruby +svc_user = User.find_by(username: 'ourServiceUser') +token = 'githubAccessToken' + +Project.where(mirror: true).each do |project| + import_url = project.import_url + + # The url we want is https://token@project/path.git + repo_url = if import_url.include?('@') + # Case 1: The url is something like https://23423432@project/path.git + import_url.split('@').last + elsif import_url.include?('//') + # Case 2: The url is something like https://project/path.git + import_url.split('//').last + end + + next unless repo_url + + final_url = "https://#{token}@#{repo_url}" + + project.mirror_user = svc_user + project.import_url = final_url + project.username_only_import_url = final_url + project.save +end +``` + +## Users + +### Finding users + +```ruby +# By username +user = User.find_by(username: '') + +# By primary email +user = User.find_by(email: '') + +# By any email (primary or secondary) +user = User.find_by_any_email('') + +# Admins +User.admins +admin = User.admins.first +``` + +### Block + +```ruby +User.find_by_username().block! +``` + +### Unblock + +```ruby +User.find_by_username().active +``` + +### Skip reconfirmation + +```ruby +user = User.find_by_username '' +user.skip_reconfirmation! +``` + +### Get an admin token + +```ruby +# Get the first admin's first access token (no longer works on 11.9+. see: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22743) +User.where(admin:true).first.personal_access_tokens.first.token + +# Get the first admin's private token (no longer works on 10.2+) +User.where(admin:true).private_token +``` + +### Create personal access token + +```ruby +personal_access_token = User.find(123).personal_access_tokens.create( + name: 'apitoken', + impersonation: false, + scopes: [:api] +) + +personal_access_token.token +``` + +You might also want to manually set the token string: + +```ruby +User.find(123).personal_access_tokens.create( + name: 'apitoken', + token_digest: Gitlab::CryptoHelper.sha256('some-token-string-here'), + impersonation: false, + scopes: [:api] +) +``` + +### Disable 2FA on a user + +```ruby +user = User.find_by_username('username') +user.disable_two_factor! +``` + +### Active users & Historical users + +```ruby +# Active users on the instance, now +User.active.count + +# The historical max on the instance as of the past year +::HistoricalData.max_historical_user_count +``` + +```bash +# Using curl and jq (up to a max 100, see [pagination](https://docs.gitlab.com/ee/api/#pagination) +curl --silent --header "Private-Token: ********************" "https://gitlab.example.com/api/v4/users?per_page=100&active" | jq --compact-output '.[] | [.id,.name,.username]' +``` + +### Block or Delete Users that have no projects or groups + +```ruby +users = User.where('id NOT IN (select distinct(user_id) from project_authorizations)') + +# How many users will be removed? +users.count + +# If that count looks sane: + +# You can either block the users: +users.each { |user| user.block! } + +# Or you can delete them: + # need 'current user' (your user) for auditing purposes +current_user = User.find_by(username: '<your username>') + +users.each do |user| + DeleteUserWorker.perform_async(current_user.id, user.id) +end +``` + +### Block Users that have no recent activity + +```ruby +days_inactive = 60 +inactive_users = User.active.where("last_activity_on <= ?", days_inactive.days.ago) + +inactive_users.each do |user| + puts "user '#{user.username}': #{user.last_activity_on}" + user.block! +end +``` + +### Find Max permissions for project/group + +```ruby +user = User.find_by_username 'username' +project = Project.find_by_full_path 'group/project' +user.max_member_access_for_project project.id +``` + +```ruby +user = User.find_by_username 'username' +group = Group.find_by_full_path 'group' +user.max_member_access_for_group group.id +``` + +## Groups + +### Count unique users in a group and sub-groups + +```ruby +group = Group.find_by_path_or_name("groupname") +members = [] +for member in group.members_with_descendants + members.push(member.user_name) +end + +members.uniq.length +``` + +```ruby +group = Group.find_by_path_or_name("groupname") + +# Count users from subgroup and up (inherited) +group.members_with_parents.count + +# Count users from parent group and down (specific grants) +parent.members_with_descendants.count +``` + +### Delete a group + +```ruby +GroupDestroyWorker.perform_async(group_id, user_id) +``` + +## LDAP + +### LDAP commands in the rails console + +TIP: **TIP:** +Use the rails runner to avoid entering the rails console in the first place. +This is great when only a single command (such as a UserSync or GroupSync) +is needed. + +```ruby +# Get debug output +Rails.logger.level = Logger::DEBUG + +# Run a UserSync (normally performed once a day) +LdapSyncWorker.new.perform + +# Run a GroupSync for all groups (9.3-) +LdapGroupSyncWorker.new.perform + +# Run a GroupSync for all groups (9.3+) +LdapAllGroupsSyncWorker.new.perform + +# Run a GroupSync for a single group (10.6-) +group = Group.find_by(name: 'my_gitlab_group') +EE::Gitlab::LDAP::Sync::Group.execute_all_providers(group) + +# Run a GroupSync for a single group (10.6+) +group = Group.find_by(name: 'my_gitlab_group') +EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) + +# Query an LDAP group directly (10.6-) +adapter = Gitlab::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +ldap_group = EE::Gitlab::LDAP::Group.find_by_cn('group_cn_here', adapter) +ldap_group.member_dns +ldap_group.member_uids + +# Query an LDAP group directly (10.6+) +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) +ldap_group.member_dns +ldap_group.member_uids + +# Lookup a particular user (10.6+) +# This could expose potential errors connecting to and/or querying LDAP that may seem to +# fail silently in the GitLab UI +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +user = Gitlab::Auth::LDAP::Person.find_by_uid('<username>',adapter) + +# Query the LDAP server directly (10.6+) +## For an example, see https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') +options = { + # the :base is required + # use adapter.config.base for the base or .group_base for the group_base + base: adapter.config.group_base, + + # :filter is optional + # 'cn' looks for all "cn"s under :base + # '*' is the search string - here, it's a wildcard + filter: Net::LDAP::Filter.eq('cn', '*'), + + # :attributes is optional + # the attributes we want to get returned + attributes: %w(dn cn memberuid member submember uniquemember memberof) +} +adapter.ldap_search(options) +``` + +### Update user accounts when the `dn` and email change + +The following will require that any accounts with the new email address are removed. +Emails have to be unique in GitLab. This is expected to work but unverified as of yet: + +```ruby +# Here's an example with a couple users. +# Each entry will have to include the old username and the new email +emails = { + 'ORIGINAL_USERNAME' => 'NEW_EMAIL_ADDRESS', + ... +} + +emails.each do |username, email| + user = User.find_by_username(username) + user.email = email + user.skip_reconfirmation! + user.save! +end + +# Run the UserSync to update the above users' data +LdapSyncWorker.new.perform +``` + +## Routes + +### Remove redirecting routes + +See <https://gitlab.com/gitlab-org/gitlab-ce/issues/41758#note_54828133>. + +```ruby +path = 'foo' +conflicting_permanent_redirects = RedirectRoute.matching_path_and_descendants(path) + +# Check that conflicting_permanent_redirects is as expected +conflicting_permanent_redirects.destroy_all +``` + +## Merge Requests + +### Find Merge Request + +```ruby +m = project.merge_requests.find_by(iid: <IID>) +m = MergeRequest.find_by_title('NEEDS UNIQUE TITLE!!!') +``` + +### Close a merge request properly (if merged but still marked as open) + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +u = User.find_by_username('') +MergeRequests::PostMergeService.new(p, u).execute(m) +``` + +### Rebase manually + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +u = User.find_by_username('') +MergeRequests::RebaseService.new(m.target_project, u).execute(m) +``` + +## CI + +### Cancel stuck pending pipelines + +See <https://gitlab.com/gitlab-com/support-forum/issues/2449#note_41929707>. + +```ruby +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').each {|p| p.cancel} +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count +``` + +### Manually modify runner minutes + +```ruby +Namespace.find_by_full_path("user/proj").namespace_statistics.update(shared_runners_seconds: 27360) +``` + +### Remove artifacts more than a week old + +```ruby +### SELECTING THE BUILDS TO CLEAR +# For a single project: +project = Project.find_by_full_path('') +builds_with_artifacts = project.builds.with_artifacts_archive + +# Prior to 10.6 the above line would be: +# builds_with_artifacts = project.builds.with_artifacts + +# Instance-wide: +builds_with_artifacts = Ci::Build.with_artifacts + +### CLEAR THEM OUT +builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago) +builds_to_clear.each do |build| + build.artifacts_expire_at = Time.now + build.erase_erasable_artifacts! +end +``` + +### Find reason failure (for when build trace is empty) (Introduced in 10.3.0) + +See <https://gitlab.com/gitlab-org/gitlab-ce/issues/41111>. + +```ruby +build = Ci::Build.find(78420) + +build.failure_reason + +build.dependencies.each do |d| { puts "status: #{d.status}, finished at: #{d.finished_at}, + completed: #{d.complete?}, artifacts_expired: #{d.artifacts_expired?}, erased: #{d.erased?}" } +``` + +### Disable strict artifact checking (Introduced in GitLab 10.3.0) + +See <https://docs.gitlab.com/ee/administration/job_artifacts.html#validation-for-dependencies>. + +```ruby +Feature.enable('ci_disable_validates_dependencies') +``` + +### Remove CI traces older than 6 months + +```ruby +current_user = User.find_by_email('cindy@gitlap.com') +Ci::Build.where("finished_at < ?", 6.months.ago.to_date).each {|b| puts b.id; b.erase(erased_by: current_user) if b.erasable?};nil +``` + +### Try CI service + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +m.project.try(:ci_service) +``` + +### Disable AutoDevOps on Existing Projects + +```ruby +Project.all.each do |p| + p.auto_devops_attributes={"enabled"=>"0"} + p.save +end +``` + +## License + +### See license plan name (since v9.3.0-ee) + +```ruby +License.current.plan +``` + +### Check if a project feature is available on the instance + +Features listed in <https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/app/models/license.rb>. + +```ruby +License.current.feature_available?(:jira_dev_panel_integration) +``` + +### Check if a project feature is available in a project + +Features listed in <https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/app/models/license.rb>. + +```ruby +p = Project.find_by_full_path('<group>/<project>') +p.feature_available?(:jira_dev_panel_integration) +``` + +### Add a license through the console + +```ruby +key = "<key>" +license = License.new(data: key) +license.save +License.current # check to make sure it applied +``` + +## Unicorn + +From [Zendesk ticket #91083](https://gitlab.zendesk.com/agent/tickets/91083) (internal) + +### Poll unicorn requests by seconds + +```ruby +require 'rubygems' +require 'unicorn' + +# Usage for this program +def usage + puts "ruby unicorn_status.rb <path to unix socket> <poll interval in seconds>" + puts "Polls the given Unix socket every interval in seconds. Will not allow you to drop below 3 second poll intervals." + puts "Example: /opt/gitlab/embedded/bin/ruby poll_unicorn.rb /var/opt/gitlab/gitlab-rails/sockets/gitlab.socket 10" +end + +# Look for required args. Throw usage and exit if they don't exist. +if ARGV.count < 2 + usage + exit 1 +end + +# Get the socket and threshold values. +socket = ARGV[0] +threshold = (ARGV[1]).to_i + +# Check threshold - is it less than 3? If so, set to 3 seconds. Safety first! +if threshold.to_i < 3 + threshold = 3 +end + +# Check - does that socket exist? +unless File.exist?(socket) + puts "Socket file not found: #{socket}" + exit 1 +end + +# Poll the given socket every THRESHOLD seconds as specified above. +puts "Running infinite loop. Use CTRL+C to exit." +puts "------------------------------------------" +loop do + Raindrops::Linux.unix_listener_stats([socket]).each do |addr, stats| + puts DateTime.now.to_s + " Active: " + stats.active.to_s + " Queued: " + stats.queued.to_s + end + sleep threshold +end +``` + +## Sidekiq + +### Size of a queue + +```ruby +Sidekiq::Queue.new('background_migration').size +``` + +### Kill a worker's Sidekiq jobs + +```ruby +queue = Sidekiq::Queue.new('repository_import') +queue.each { |job| job.delete if <condition>} +``` + +### Enable debug logging of Sidekiq + +```ruby +gitlab_rails['env'] = { + 'SIDEKIQ_LOG_ARGUMENTS' => "1" +} +``` + +Then `gitlab-ctl reconfigure; gitlab-ctl restart sidekiq`. The Sidekiq logs will now include additional data for troubleshooting. + +### Sidekiq kill signals + +See <https://github.com/mperham/sidekiq/wiki/Signals#ttin>. + +## Redis + +### Connect to redis (omnibus) + +```sh +/opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.socket +``` + +### Connect to redis (HA) + +```sh +/opt/gitlab/embedded/bin/redis-cli -h <host ip> -a <password> +``` + +## LFS + +### Get info about LFS objects and associated project + +```ruby +o=LfsObject.find_by(oid: "<oid>") +p=Project.find(LfsObjectsProject.find_by_lfs_object_id(o.id).project_id) +``` + +You can then delete these records from the database with: + +```ruby +LfsObjectsProject.find_by_lfs_object_id(o.id).destroy +o.destroy +``` + +You would also want to combine this with deleting the LFS file in the LFS storage +area on disk. It remains to be seen exactly how or whether the deletion is useful, however. + +## Decryption Problems + +### Bad Decrypt Script (for encrypted variables) + +See <https://gitlab.com/snippets/1730735/raw>. + +This script will go through all the encrypted variables and count how many are not able +to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json` +file is most up to date: + +```bash +wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw +gitlab-rails runner /tmp/bad-decrypt.rb +``` + +If `ProjectImportData Bad count:` is detected and the decision is made to delete the +encrypted credentials to allow manual reentry: + +```ruby + # Find the ids of the corrupt ProjectImportData objects + total = 0 + bad = [] + ProjectImportData.find_each do |data| + begin + total += 1 + data.credentials + rescue => e + bad << data.id + end + end + + puts "Bad count: #{bad.count} / #{total}" + + # See the bad ProjectImportData ids + bad + + # Remove the corrupted credentials + import_data = ProjectImportData.where(id: bad) + import_data.each do |data| + data.update_columns({ encrypted_credentials: nil, encrypted_credentials_iv: nil, encrypted_credentials_salt: nil}) + end +``` + +If `User OTP Secret Bad count:` is detected. For each user listed disable/enable +two-factor authentication. + +### Decrypt Script for encrypted tokens + +This script will search for all encrypted tokens that are causing decryption errors, +and update or reset as needed: + +```bash +wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw +gitlab-rails runner /tmp/encrypted-tokens.rb +``` + +## Geo + +### Artifacts + +#### Find failed artifacts + +```ruby +Geo::JobArtifactRegistry.failed +``` + +#### Download artifact + +```ruby +Gitlab::Geo::JobArtifactDownloader.new(:job_artifact, <artifact_id>).execute +``` + +#### Get a count of the synced artifacts + +```ruby +Geo::JobArtifactRegistry.synced.count +``` + +#### Find `ID` of synced artifacts that are missing on primary + +```ruby +Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) +``` + +### Repository verification failures + +#### Get the number of verification failed repositories + +```ruby +Geo::ProjectRegistryFinder.new.count_verification_failed_repositories +``` + +#### Find the verification failed repositories + +```ruby +Geo::ProjectRegistry.verification_failed_repos +``` + +### Find repositories that failed to sync + +```ruby +Geo::ProjectRegistryFinder.new.find_failed_project_registries('repository') +``` + +### Resync repositories + +#### Queue up all repositories for resync. Sidekiq will handle each sync + +```ruby +Geo::ProjectRegistry.update_all(resync_repository: true, resync_wiki: true) +``` + +#### Sync individual repository now + +```ruby +project = Project.find_by_full_path('<group/project>') + +Geo::RepositorySyncService.new(project).execute +``` diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md new file mode 100644 index 00000000000..2bbb498f020 --- /dev/null +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -0,0 +1,339 @@ +--- +type: reference +--- + +# Linux Cheat Sheet + +This is the GitLab Support Team's collection of information regarding Linux, that they +sometimes use while troubleshooting. It is listed here for transparency, +and it may be useful for users with experience with Linux. If you are currently +having an issue with GitLab, you may want to check your [support options](https://about.gitlab.com/support/) +first, before attempting to use this information. + +CAUTION: **CAUTION:** +If you are administering GitLab you are expected to know these commands for your distribution +of choice. If you are a GitLab Support Engineer, consider this a cross-reference to +translate `yum` -> `apt-get` and the like. + +Note: **Note:** +Most of the commands below have not been labeled as to which distribution they work +on. Contributions are welcome to help add them. + +## System Commands + +### Distro Information + +```bash +# Debian/Ubuntu +uname -a +lsb_release -a + +# CentOS/RedHat +cat /etc/centos-release +cat /etc/redhat-release + +# This will provide a lot more information +cat /etc/os-release +``` + +### Shut down or Reboot + +```bash +shutdown -h now +reboot +``` + +### Permissions + +```bash +# change the user:group ownership of a file/dir +chown root:git <file_or_dir> + +# make a file executable +chmod u+x <file> +``` + +### Files & Dirs + +```bash +# create a new directory and all subdirectories +mkdir -p dir/dir2/dir3 + +# Send a command's output to file.txt, no STDOUT +ls > file.txt + +# Send a command's output to file.txt AND see it in STDOUT +ls | tee /tmp/file.txt + +# Search and Replace within a file +sed -i 's/original-text/new-text/g' <filename> +``` + +### See all set environment variables + +```bash +env +``` + +## Searching + +### File names + +```bash +# search for a file in a filesystem +find . -name 'filename.rb' -print + +# locate a file +locate <filename> + +# see command history +history + +# search CLI history +<ctrl>-R +``` + +### File contents + +```bash +# -B/A = show 2 lines before/after search_term +grep -B 2 -A 2 search_term <filename> + +# -<number> shows both before and after +grep -2 search_term <filename> + +# Search on all files in directory (recursively) +grep -r search_term <directory> + +# search through *.gz files is the same except with zgrep +zgrep search_term <filename> + +# Fast grep printing lines containing a string pattern +fgrep -R string_pattern <filename or directory> +``` + +### CLI + +```bash +# View command history +history + +# Run last command that started with 'his' (3 letters min) +!his + +# Search through command history +<ctrl>-R + +# Execute last command with sudo +sudo !! +``` + +## Managing resources + +### Memory, Disk, & CPU usage + +```bash +# disk space info. The '-h' gives the data in human-readable values +df -h + +# size of each file/dir and its contents in the current dir +du -hd 1 + +# or alternative +du -h --max-depth=1 + +# find files greater than certain size(k, M, G) and list them in order +# get rid of the + for exact, - for less than +find / -type f -size +100M -print0 | xargs -0 du -hs | sort -h + +# Find free memory on a system +free -m + +# Find what processes are using memory/CPU and organize by it +# Load average is 1/CPU for 1, 5, and 15 minutes +top -o %MEM +top -o %CPU +``` + +### Strace + +```bash +# strace a process +strace -tt -T -f -y -s 1024 -p <pid> + +# -tt print timestamps with microsecond accuracy + +# -T print the time spent in each syscall + +# -f also trace any child processes that forked + +# -y print the path associated with file handles + +# -s max string length to print for an event + +# -o output file + +# run strace on all unicorn processes +ps auwx | grep unicorn | awk '{ print " -p " $2}' | xargs strace -tt -T -f -y -s 1024 -o /tmp/unicorn.txt +``` + +See the [strace zine](https://wizardzines.com/zines/strace/) for a quick walkthrough. + +Brendan Gregg has a more detailed explanation of [how to use strace](http://www.brendangregg.com/blog/2014-05-11/strace-wow-much-syscall.html). + +Be aware that strace can have major impacts to system performance when it is running. + +### The Strace Parser tool + +Our [strace-parser tool](https://gitlab.com/wchandler/strace-parser) can be used to +provide a high level summary of the `strace` output. It is similar to `strace -C`, +but provides much more detailed statistics. + +MacOS and Linux binaries [are available](https://gitlab.com/gitlab-com/support/toolbox/strace-parser/-/tags), +or you can build it from source if you have the Rust compiler. + +#### How to use the tool + +First run the tool with no arguments other than the strace output file name to get +a summary of the top processes sorted by time spent actively performing tasks. You +can also sort based on total time, # of syscalls made, PID #, and # of child processes +using the `-S` or `--sort` flag. The number of results defaults to 25 processes, but +can be changed using the `-c`/`--count` option. See `--help` for full details. + +```sh +$ ./strace-parser strace.txt + +Top 25 PIDs +----------- + + pid active (ms) wait (ms) total (ms) % active syscalls + ---------- ---------- --------- --------- --------- --------- + 8795 689.072 45773.832 46462.902 16.89% 23018 + 13408 679.432 55910.891 56590.320 16.65% 28593 + 6423 554.822 13175.485 13730.308 13.60% 13735 +... +``` + +Based on the summary, you can then view the details of syscalls made by one or more +procsses using the `-p`/`--pid` for a specific process, or `-s`/`--stats` flags for +a sorted list. `--stats` takes the same sorting and count options as summary. + +```sh +$ ./strace-parse strace.text -p 6423 + +PID 6423 +13735 syscalls, active time: 554.822ms, total time: 13730.308ms + + syscall count total max avg min errors + (ms) (ms) (ms) (ms) + --------------- -------- ---------- ---------- ---------- ---------- -------- + epoll_wait 628 13175.485 21.259 20.980 0.020 + clock_gettime 7326 199.500 0.249 0.027 0.013 + stat 2101 110.768 19.056 0.053 0.017 ENOENT: 2076 + ... + --------------- + + Parent PID: 495 + Child PIDs: 8383, 8418, 8419, 8420, 8421 + + Slowest file access times for PID 6423: + + open (ms) timestamp error file name + ----------- --------------- --------------- ---------- + 29.818 10:53:11.528954 /srv/gitlab-data/builds/2018_08/6174/954448.log + 12.309 10:53:46.708274 /srv/gitlab-data/builds/2018_08/5342/954186.log + 0.039 10:53:49.222110 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_note.html.haml + 0.035 10:53:49.125115 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_push.html.haml + ... +``` + +In the example above, we can see that file opening times on `/srv/gitlab-data` are +extremely slow, about 100X slower than `/opt/gitlab`. + +When nothing stands out in the results, a good way to get more context is to run `strace` +on your own GitLab instance while performing the action performed by the customer, +then compare summaries of both results and dive into the differences. + +#### Stats for the open syscall + +Rough numbers for calls to `open` and `openat` (used to access files) on various configurations. +Slow storage can cause the dreaded `DeadlineExceeded` error in Gitaly. + +Also [see this entry](https://docs.gitlab.com/ee/administration/operations/filesystem_benchmarking.html) +in the handbook for quick tests customers can perform to check their filesystem performance. + +Keep in mind that timing information from `strace` is often somewhat inaccurate, so +small differences should not be considered significant. + +|Setup | access times | +|:--------------|:--------------| +| EFS | 10 - 30ms | +| Local Storage | 0.01 - 1ms | + +## Networking + +### Ports + +```bash +# Find the programs that are listening on ports +netstat -plnt +ss -plnt +lsof -i -P | grep <port> +``` + +### Internet/DNS + +```bash +# Show domain IP address +dig +short example.com +nslookup example.com + +# Check DNS using specific nameserver +# 8.8.8.8 = google, 1.1.1.1 = cloudflare, 208.67.222.222 = opendns +dig @8.8.8.8 example.com +nslookup example.com 1.1.1.1 + +# Find host provider +whois <ip_address> | grep -i "orgname\|netname" + +# Curl headers with redirect +curl --head --location https://example.com +``` + +## Package Management + +```bash +# Debian/Ubuntu + +# List packages +dpkg -l +apt list --installed + +# Find an installed package +dpkg -l | grep <package> +apt list --installed | grep <package> + +# Install a package +dpkg -i <package_name>.deb +apt-get install <package> +apt install <package> + +# CentOS/RedHat + +# Install a package +yum install <package> +dnf install <package> # RHEL/CentOS 8+ + +rpm -ivh <package_name>.rpm + +# Find an installed package +rpm -qa | grep <package> +``` + +## Logs + +```bash +# Print last lines in log file where 'n' +# is the number of lines to print +tail -n /path/to/log/file +``` diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md new file mode 100644 index 00000000000..075effc5dc3 --- /dev/null +++ b/doc/administration/troubleshooting/test_environments.md @@ -0,0 +1,126 @@ +--- +type: reference +--- + +# Apps for a Testing Environment + +This is the GitLab Support Team's collection of information regarding testing environments, +for use while troubleshooting. It is listed here for transparency, and it may be useful +for users with experience with these tools. If you are currently having an issue with +GitLab, you may want to check your [support options](https://about.gitlab.com/support/) +first, before attempting to use this information. + +NOTE: **Note:** +This page was initially written for Support Engineers, so some of the links +are only available internally at GitLab. + +## Docker + +The following were tested on docker containers running in the cloud. Support Engineers, +please see [these docs](https://gitlab.com/gitlab-com/dev-resources/tree/master/dev-resources#running-docker-containers) +on how to run Docker containers on `dev-resources`. Other setups haven't been tested, +but contributions are welcome. + +### GitLab + +Please see [our Docker test environment docs](https://docs.gitlab.com/ee/install/digitaloceandocker.html#create-new-gitlab-container) +for how to run GitLab on Docker. When spinning this up with `docker-machine`, ensure +you change a few things: + +1. Update the name of the `docker-machine` host. You can see a list of hosts + with `docker-machine ls`. +1. Expose the necessary ports using the `-p` flag. Docker normally doesn't + allow access to any ports it uses outside of the container, so they must be + explicitly exposed. +1. Add any necessary `gitlab.rb` configuration to the + `GITLAB_OMNIBUS_CONFIG` variable. + +For example, when the `docker-machine` host we want to use is `do-docker`: + +```sh +docker run --detach --name gitlab \ +--env GITLAB_OMNIBUS_CONFIG="external_url 'http://$(docker-machine ip do-docker)'; gitlab_rails['gitlab_shell_ssh_port'] = 2222;" \ +--hostname $(docker-machine ip do-docker) \ +-p 80:80 -p 2222:22 \ +gitlab/gitlab-ee:11.5.3-ee.0 +``` + +### SAML + +#### SAML for Authentication + +We can use the [test-saml-idp Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp) +to do the work for us: + +```sh +docker run --name gitlab_saml -p 8080:8080 -p 8443:8443 \ +-e SIMPLESAMLPHP_SP_ENTITY_ID=<GITLAB_IP_OR_DOMAIN> \ +-e SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=<GITLAB_IP_OR_DOMAIN>/users/auth/saml/callback \ +-d jamedjo/test-saml-idp +``` + +The following will also need to go in your `/etc/gitlab/gitlab.rb`. See [our SAML docs](https://docs.gitlab.com/ee/integration/saml.html) +for more, as well as the list of [default usernames, passwords, and emails](https://hub.docker.com/r/jamedjo/test-saml-idp/#usage). + +```ruby +gitlab_rails['omniauth_enabled'] = true +gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] +gitlab_rails['omniauth_sync_email_from_provider'] = 'saml' +gitlab_rails['omniauth_sync_profile_from_provider'] = ['saml'] +gitlab_rails['omniauth_sync_profile_attributes'] = ['email'] +gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml' +gitlab_rails['omniauth_block_auto_created_users'] = false +gitlab_rails['omniauth_auto_link_ldap_user'] = false +gitlab_rails['omniauth_auto_link_saml_user'] = true +gitlab_rails['omniauth_providers'] = [ + { + "name" => "saml", + "label" => "SAML", + "args" => { + assertion_consumer_service_url: '<GITLAB_IP_OR_DOMAIN>/users/auth/saml/callback', + idp_cert_fingerprint: '119b9e027959cdb7c662cfd075d9e2ef384e445f', + idp_sso_target_url: '<SAML_IP_OR_DOMAIN>:8080/simplesaml/saml2/idp/SSOService.php', + issuer: '<GITLAB_IP_OR_DOMAIN>', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' + } + } +] +``` + +#### GroupSAML for GitLab.com + +See [the GDK SAML documentation](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/saml.md). + +### ElasticSearch + +```sh +docker run -d --name elasticsearch \ +-p 9200:9200 -p 9300:9300 \ +-e "discovery.type=single-node" \ +docker.elastic.co/elasticsearch/elasticsearch:5.5.1 +``` + +Then confirm it works in the browser at `curl http://<IP_ADDRESS>:9200/_cat/health`. +ElasticSearch's default username is `elastic` and password is `changeme`. + +### PlantUML + +See [our PlantUML docs](../integration/plantuml.md#docker) +on running PlantUML in Docker. + +### Jira + +```sh +docker run -d -p 8081:8080 cptactionhank/atlassian-jira:latest +``` + +Then go to `<IP_ADDRESS>:8081` in the browser to set it up. This requires a +Jira license. + +### Grafana + +```sh +docker run -d --name grafana -e "GF_SECURITY_ADMIN_PASSWORD=gitlab" -p 3000:3000 grafana/grafana +``` + +Access it at `<IP_ADDRESS>:3000`. diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index bfba21646b5..8f2e95dbb10 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -116,6 +116,7 @@ The following table lists available parameters for jobs: | [`extends`](#extends) | Configuration entries that this job is going to inherit from. | | [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | | [`variables`](#variables) | Define job variables on a job level. | +| [interruptible](#interruptible) | Defines if a job can be canceled when made redundant by a newer run | NOTE: **Note:** Parameters `types` and `type` are [deprecated](#deprecated-parameters). @@ -2083,6 +2084,46 @@ staging: branch: stable ``` +### `interruptible` + +`interruptible` is used to indicate that a job should be canceled if made redundant by a newer run of the same job. Defaults to `false` if there is an environment defined and `true` otherwise. +This value will only be used if the [automatic cancellation of redundant pipelines feature](https://docs.gitlab.com/ee/user/project/pipelines/settings.html#auto-cancel-pending-pipelines) +is enabled. + +When enabled, a pipeline on the same branch will be canceled when: + +- It is made redundant by a newer pipeline run. +- Either all jobs are set as interruptible, or any uninterruptible jobs are not yet pending. + +Pending jobs are always considered interruptible. + +TIP: **Tip:** +Set jobs as uninterruptible that should behave atomically and should never be canceled once started. + +Here is a simple example: + +```yaml +stages: + - stage1 + - stage2 + +step-1: + stage: stage1 + script: + - echo "Can be canceled" + +step-2: + stage: stage2 + script: + - echo "Can not be canceled" + interruptible: false +``` + +In the example above, a new pipeline run will cause an existing running pipeline to be: + +- Canceled, if only `step-1` is running or pending. +- Not canceled, once `step-2` becomes pending. + ### `include` > - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 80890d96159..bcfc0734c06 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -116,8 +116,13 @@ warrant a comment could be: - Any benchmarking performed to complement the change - Potentially insecure code -Do not add these comments directly to the source code, unless the -reviewer requires you to do so. +Avoid: + +- Adding comments (referenced above, or TODO items) directly to the source code unless the reviewer requires you to do so. If the comments are added due to an actionable task, +a link to an issue must be included. +- Assigning merge requests with failed tests to maintainers. If the tests are failing and you have to assign, ensure you leave a comment with an explanation. +- Excessively mentioning maintainers through email or Slack (if the maintainer is reachable +through Slack). If you can't assign a merge request, `@` mentioning a maintainer in a comment is acceptable and in all other cases assigning the merge request is sufficient. This [saves reviewers time and helps authors catch mistakes earlier](https://www.ibm.com/developerworks/rational/library/11-proven-practices-for-peer-review/index.html#__RefHeading__97_174136755). diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 719b9aa212a..c267669aa79 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -463,7 +463,7 @@ The current tests are: to [check locally](#previewing-the-changes-live) before pushing to GitLab by executing the command `bundle exec nanoc check internal_links` on your local [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. In addition, - `docs-lint` also runs [markdownlint](styleguide.md#markdown-rules) to ensure the + `docs-lint` also runs [`markdownlint`](#markdownlint) to ensure the markdown is consistent across all documentation. 1. [`ee_compat_check`](../automatic_ce_ee_merge.md#avoiding-ce-ee-merge-conflicts-beforehand) (runs on CE only): When you submit a merge request to GitLab Community Edition (CE), @@ -488,7 +488,7 @@ help you catch common issues before raising merge requests for review of documen The following are some suggested linters you can install locally and sample configuration: - [`proselint`](#proselint) -- [`markdownlint`](#markdownlint) +- [`markdownlint`](#markdownlint), which is the same as the test run in [`docs-lint`](#testing) NOTE: **Note:** This list does not limit what other linters you can add to your local documentation writing toolchain. @@ -534,76 +534,56 @@ A file with `proselint` configuration must be placed in a #### `markdownlint` -`markdownlint` checks that certain rules ([example](https://github.com/DavidAnson/markdownlint/blob/master/README.md#rules--aliases)) -are followed for Markdown syntax. Our [Documentation Style Guide](styleguide.md) and -[Markdown Guide](https://about.gitlab.com/handbook/product/technical-writing/markdown-guide/) +[`markdownlint`](https://github.com/DavidAnson/markdownlint) checks that markdown +syntax follows [certain rules](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules), +and is used by the [`docs-lint` test](#testing) with a [configuration file](#markdownlint-configuration). +Our [Documentation Style Guide](styleguide.md#markdown) and [Markdown Guide](https://about.gitlab.com/handbook/product/technical-writing/markdown-guide/) elaborate on which choices must be made when selecting Markdown syntax for GitLab -documentation. This tool helps catch deviations from those guidelines, and matches the -tests run on the documentation by [`docs-lint`](#testing). +documentation. This tool helps catch deviations from those guidelines. `markdownlint` can be used [on the command line](https://github.com/igorshubovych/markdownlint-cli#markdownlint-cli--), either on a single Markdown file or on all Markdown files in a project. For example, to run `markdownlint` on all documentation in the [`gitlab-ce` project](https://gitlab.com/gitlab-org/gitlab-ce), -run the following commands from within the `gitlab-ce` project: +run the following commands from within your `gitlab-ce` project root directory, which will +automatically detect the [`.markdownlint.json`](#markdownlint-configuration) config +file in the root of the project, and test all files in `/doc` and its subdirectories: ```sh -cd doc -markdownlint **/*.md +markdownlint 'doc/**/*.md' ``` -`markdownlint` can also be run from within editors using plugins. For example, the following plugins - are available: +If you wish to use a different config file, use the `-c` flag: + +```sh +markdownlint -c <config-file-name> 'doc/**/*.md' +``` + +`markdownlint` can also be run from within text editors using [plugins/extensions](https://github.com/DavidAnson/markdownlint#related), +such as: - [Sublime Text](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint) - [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint) -- [Others](https://github.com/DavidAnson/markdownlint#related) +- [Atom](https://atom.io/packages/linter-node-markdownlint) -##### Sample `markdownlint` configuration +It is best to use the [same configuration file](#markdownlint-configuration) as what +is in use in the four repos that are the sources for <https://docs.gitlab.com>. Each +plugin/extension has different requirements regarding the configuration file, which +is explained in each editor's docs. -The following sample `markdownlint` configuration modifies the available default rules to: +##### `markdownlint` configuration -- Adhere to the [Documentation Style Guide](styleguide.md). -- Apply conventions found in the GitLab documentation. -- Allow the flexibility of using some inline HTML. +Each formatting issue that `markdownlint` checks has an associated +[rule](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules). +These rules are configured in the `.markdownlint.json` files located in the root of +four repos that are the sources for <https://docs.gitlab.com>: -```json -{ - "default": true, - "header-style": { "style": "atx" }, - "ul-style": { "style": "dash" }, - "line-length": false, - "no-trailing-punctuation": false, - "ol-prefix": { "style": "one" }, - "blanks-around-fences": true, - "no-inline-html": { - "allowed_elements": [ - "table", - "tbody", - "tr", - "td", - "ul", - "ol", - "li", - "br", - "img", - "a", - "strong", - "i", - "div", - "b" - ] - }, - "hr-style": { "style": "---" }, - "code-block-style": { "style": "fenced" }, - "fenced-code-language": false, - "no-duplicate-header": { "allow_different_nesting": true }, - "commands-show-output": false -} -``` +- <https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.markdownlint.json> +- <https://gitlab.com/gitlab-org/gitlab-runner/blob/master/.markdownlint.json> +- <https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/.markdownlint.json> +- <https://gitlab.com/charts/gitlab/blob/master/.markdownlint.json> -For [`markdownlint`](https://github.com/DavidAnson/markdownlint/), this configuration must be -placed in a [valid location](https://github.com/igorshubovych/markdownlint-cli#configuration). For -example, `~/.markdownlintrc`. +By default all rules are enabled, so the configuration file is used to disable unwanted +rules, and also to configure optional parameters for enabled rules as needed. ## Danger Bot diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 753a636a779..be4d5b5353f 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -4,30 +4,74 @@ description: "Learn how GitLab docs' global navigation works and how to add new # Global navigation -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/362) -in GitLab 11.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/362) in GitLab 11.6. > - [Updated](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/482) in GitLab 12.1. +> - [Per-project](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/498) navigation added in GitLab 12.2. -The global nav adds to the left sidebar the ability to -navigate and explore the contents of GitLab's documentation. +Global navigation (the left-most pane in our three pane documentation) provides: -The global nav should be maintained consistent through time to allow the -users to locate their most-visited links easily to facilitate navigation. -Therefore, any updates must be carefully considered by the technical writers. +- A high-level grouped view of product features. +- The ability to discover new features by browsing the menu structure. +- A way to allow the reader to focus on product areas. +- The ability to refine landing pages, so they don't have to do all the work of surfacing + every page contained within the documentation. -## Adding new items to the global nav +## Adding new items -To add a new doc to the nav, first and foremost, check with the technical writing team: +All new pages need a new navigation item. Without a navigation, the page becomes "orphaned". That +is: -- If it's applicable -- What's the exact position the doc will be added to the nav +- The navigation shuts when the page is opened, and the reader loses their place. +- The page doesn't belong in a group with other pages. -Once you get their approval and their guidance in regards to the position on the nav, -read trhough this page to understand how it works, and submit a merge request to the -docs site, adding the doc you wish to include in the nav into the -[global nav data file](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/global-nav.yaml). +This means the decision to create a new page is a decision to create new navigation item and vice +versa. -Don't forget to ask a technical writer to review your changes before merging. +### Where to add + +Documentation pages can be said to belong in the following groups: + +- GitLab users. This is documentation for day-to-day use of GitLab for users with any level + of permissions, from Reporter to Owner. +- GitLab administrators. This tends to be documentation for self-managed instances that requires + access to the underlying infrastructure hosting GitLab. +- Other documentation. This includes documentation for customers outside their day-to-day use of + GitLab and for contributors. Documentation that doesn't fit in the other groups belongs here. + +With these groups in mind, the following are general rules for where new items should be added. + +- User documentation for: + - Group-level features belongs under **Groups**. + - Project-level features belongs under **Projects**. + - Features outside a group or project level (sometimes called "instance-level") can be placed at + the top-level, but care must be taken not to overwhelm that top-level space. If possible, such + features could be grouped in some way. + - Outside the above, most other miscellaneous user documentation belongs under **User**. +- Administration documentation belongs under **Administrator**. +- Other documentation belongs at the top-level, but care must be taken to not create an enormously + long top-level navigation, which defeats the purpose of it. + +NOTE: **Note:** +Making all documentation and navigation items adhere to these principles is being progressively +rolled out. + +### What to add + +Having decided where to add a navigation element, the next step is deciding what to add. The +mechanics of what is required is [documented below](#data-file) but, in principle: + +- Navigation item text (that which the reader sees) should: + - Be as short as possible. + - Be contextual. It's rare to need to repeat text from a parent item. + - Avoid jargon or terms of art, unless ubiquitous. For example, **CI** is an acceptable + substitution for **Continuous Integration**. +- Navigation links must follow the rules documented in the [data file](#data-file). +- EE badging is subject to the following: + - Required when linking to an EE-only page. + - Not required when linking to a page that is a mix of CE and EE-only content. + - Required when all sub-items are EE-only. In this case, no sub-items are EE badged. + - Not required when sub-items are a mix of CE and EE-only items. In this case, each item is + badged appropriately. ## How it works @@ -55,7 +99,7 @@ To see the improvements planned, check the [global nav epic](https://gitlab.com/groups/gitlab-com/-/epics/21). CAUTION: **Attention!** -**Do not** [add items](#adding-new-items-to-the-global-nav) to the global nav without +**Do not** [add items](#adding-new-items) to the global nav without the consent of one of the technical writers. ## Composition @@ -70,8 +114,13 @@ the data among the nav in containers properly [styled](#css-classes). ### Data file -The [data file](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/global-nav.yaml) -is structured in three components: sections, categories, and docs. +The data file describes the structure of the navigation for the applicable project. All data files +are stored at <https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/> and comprise +three components: + +- Sections +- Categories +- Docs #### Sections diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 1aef0ed855c..bb598906967 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -66,10 +66,12 @@ the GitLab Documentation website. - [Google Analytics](https://marketingplatform.google.com/about/analytics/) - [Google Tag Manager](https://developers.google.com/tag-manager/) -## Global nav +## Global navigation -To understand how the global nav (left sidebar) is built, please -read through the [global navigation](global_nav.md) doc. +Read through the global navigation](global_nav.md) documentation to understand: + +- How the global navigation is built. +- How to add new navigation items. ## Deployment diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 283e8bea8d5..09dd31e2aee 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -110,18 +110,11 @@ Hard-coded HTML is valid, although it's discouraged to be used while we have `/h ### Markdown Rules GitLab ensures that the Markdown used across all documentation is consistent, as -well as easy to review and maintain, by testing documentation changes with -[Markdownlint (mdl)](https://github.com/markdownlint/markdownlint). This lint test -checks many common problems with Markdown, and fails when any document has an issue +well as easy to review and maintain, by [testing documentation changes](index.md#testing) with +[`markdownlint`](index.md#markdownlint). This lint test fails when any document has an issue with Markdown formatting that may cause the page to render incorrectly within GitLab. It will also fail when a document is using non-standard Markdown (which may render -correctly, but is not the current standard in GitLab documentation). - -Each formatting issue that mdl checks has an associated [rule](https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md), -and the rules that are currently enabled for GitLab documentation are listed in the -[`.mdlrc.style`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.mdlrc.style) file. -Configuration options are set in the [`.mdlrc`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.mdlrc.style) -file. +correctly, but is not the current standard for GitLab documentation). ## Structure @@ -460,7 +453,7 @@ For other punctuation rules, please refer to the This is to ensure that no document with wrong heading is going live without an audit, thus preventing dead links and redirection issues when corrected. -- Leave exactly one new line after a heading. +- Leave exactly one blank line before and after a heading. - Do not use links in headings. - Add the corresponding [product badge](#product-badges) according to the tier the feature belongs. diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index e1350b02262..9ac87a17232 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -10,8 +10,10 @@ The following are required to install and test the app: For the app to work, Jira Cloud should be able to connect to the GitLab instance through the internet. - To easily expose your local development environment, you can use tools like [serveo](https://serveo.net) or [ngrok](https://ngrok.com). - These also take care of SSL for you because Jira requires all connections to the app host to be over SSL. + To easily expose your local development environment, you can use tools like + [serveo](https://medium.com/@osanda.deshan/how-to-forward-my-local-port-to-public-using-serveo-4979f352a3bf) + or [ngrok](https://ngrok.com). These also take care of SSL for you because Jira + requires all connections to the app host to be over SSL. > This feature is currently behind the `:jira_connect_app` feature flag diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 89500ef9a90..2200069ecfd 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -1,5 +1,7 @@ # Best practices when writing end-to-end tests +## Avoid using a GUI when it's not required + The majority of the end-to-end tests require some state to be built in the application for the tests to happen. A good example is a user being logged in as a pre-condition for testing the feature. @@ -36,3 +38,18 @@ Finally, interacting with the application only by its GUI generates a higher rat - When depending only on the GUI to create the application's state and tests fail due to front-end issues, we can't rely on the test failures rate, and we generate a higher rate of test flakiness. Now that we are aware of all of it, [let's go create some tests](quick_start_guide.md). + +## Prefer to split tests across multiple files + +Our framework includes a couple of parallelization mechanisms that work by executing spec files in parallel. + +However, because tests are parallelized by spec *file* and not by test/example, we can't achieve greater parallelization if a new test is added to an existing file. + +Nonetheless, there could be other reasons to add a new test to an existing file. + +For example, if tests share state that is expensive to set up it might be more efficient to perform that setup once even if it means the tests that use the setup can't be parallelized. + +In summary: + +- **Do**: Split tests across separate files, unless the tests share expensive setup. +- **Don't**: Put new tests in an existing file without considering the impact on parallelization. diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index f4cee410066..00371057d3c 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -88,7 +88,7 @@ class RenameUsersUpdatedAtToUpdatedAtTimestamp < ActiveRecord::Migration[4.2] end def down - cleanup_concurrent_column_rename :users, :updated_at_timestamp, :updated_at + undo_rename_column_concurrently :users, :updated_at, :updated_at_timestamp end end ``` @@ -118,7 +118,7 @@ class CleanupUsersUpdatedAtRename < ActiveRecord::Migration[4.2] end def down - rename_column_concurrently :users, :updated_at_timestamp, :updated_at + undo_cleanup_concurrent_column_rename :users, :updated_at, :updated_at_timestamp end end ``` diff --git a/doc/security/reset_root_password.md b/doc/security/reset_root_password.md index ec360e2d338..00c9dc1407d 100644 --- a/doc/security/reset_root_password.md +++ b/doc/security/reset_root_password.md @@ -9,7 +9,7 @@ To reset your root password, first log into your server with root privileges. Start a Ruby on Rails console with this command: ```bash -gitlab-rails console production +gitlab-rails console -e production ``` Wait until the console has loaded. diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png Binary files differindex 61f683c1335..1fe76a9e08f 100644..100755 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png diff --git a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png Binary files differindex 0b2dfecd9e7..09979ba99b3 100644..100755 --- a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png +++ b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard.png Binary files differdeleted file mode 100644 index baa136fd885..00000000000 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png Binary files differnew file mode 100755 index 00000000000..51e80bdb50d --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index a98ca1fb338..ac539509e22 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -52,7 +52,7 @@ At the project level, the Security Dashboard displays the latest security report for your project. Use it to find and fix vulnerabilities affecting the [default branch](../../project/repository/branches/index.md#default-branch). - + ## Group Security Dashboard @@ -71,12 +71,15 @@ Once you're on the dashboard, at the top you should see a series of filters for: - Report type - Project +To the right of the filters, you should see a **Hide dismissed** toggle button. + NOTE: **Note:** The dashboard only shows projects with [security reports](#supported-reports) enabled in a group.  -Selecting one or more filters will filter the results in this page. +Selecting one or more filters will filter the results in this page. Disabling the **Hide dismissed** +toggle button will let you also see vulnerabilities that have been dismissed. The main section is a list of all the vulnerabilities in the group, sorted by severity. In that list, you can see the severity of the vulnerability, its name, its diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 40ed0db4c57..83ddcf61664 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -82,10 +82,12 @@ certificates are valid and up-to-date. NOTE: **Note:** The -[stable/cert-manager](https://github.com/helm/charts/tree/master/stable/cert-manager) +[jetstack/cert-manager](https://github.com/jetstack/cert-manager) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/vendor/cert_manager/values.yaml) -file. +file. Prior to GitLab 12.3, +the [stable/cert-manager](https://github.com/helm/charts/tree/master/stable/cert-manager) +chart was used. ### GitLab Runner diff --git a/doc/user/permissions.md b/doc/user/permissions.md index c28a5e49ec4..02b5f7437ee 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -37,6 +37,8 @@ usernames. A GitLab administrator can configure the GitLab instance to NOTE: **Note:** In GitLab 11.0, the Master role was renamed to Maintainer. +While Maintainer is the highest project-level role, some actions can only be performed by a personal namespace or group owner. + The following table depicts the various user permission levels in a project. | Action | Guest | Reporter | Developer |Maintainer| Owner | @@ -74,7 +76,7 @@ The following table depicts the various user permission levels in a project. | View project statistics | | ✓ | ✓ | ✓ | ✓ | | View Error Tracking list | | ✓ | ✓ | ✓ | ✓ | | Pull from [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish to [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ || +| Publish to [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | | Push to non-protected branches | | | ✓ | ✓ | ✓ | diff --git a/doc/user/project/import/img/import_projects_from_repo_url.png b/doc/user/project/import/img/import_projects_from_repo_url.png Binary files differindex c453c7e558a..90bcff5d31b 100644 --- a/doc/user/project/import/img/import_projects_from_repo_url.png +++ b/doc/user/project/import/img/import_projects_from_repo_url.png diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index d6da8cb99c7..9f31f38460a 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -289,6 +289,7 @@ as pushing changes: - Set the merge request to remove the source branch when it's merged. - Set the title of the merge request to a particular title. - Set the description of the merge request to a particular description. +- Add or remove labels from the merge request. ### Create a new merge request using git push options @@ -375,6 +376,33 @@ git push -o merge_request.description="The description I want" You can also use this push option in addition to the `merge_request.create` push option. +### Add or remove labels using git push options + +You can add or remove labels from merge requests using push options. + +For example, to add two labels to an existing merge request, use the +`merge_request.label` push option: + +```sh +git push -o merge_request.label="label1" -o merge_request.label="label2" +``` + +To remove two labels from an existing merge request, use +the `merge_request.unlabel` push option: + +```sh +git push -o merge_request.unlabel="label1" -o merge_request.unlabel="label2" +``` + +You can also use these push options in addition to the +`merge_request.create` push option. + +To create a merge request and add two labels to it, use: + +```sh +git push -o merge_request.create -o merge_request.label="label1" -o merge_request.label="label2" +``` + ## Find the merge request that introduced a change > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/2383) in GitLab 10.5. diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index 8423b962948..895c8ac88e7 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -15,7 +15,7 @@ By default, a protected branch does four simple things: - It prevents its creation, if not already created, from everybody except users with Maintainer permission. -- It prevents pushes from everybody except users with Maintainer permission. +- It prevents pushes from everybody except users with **Allowed** permission. - It prevents **anyone** from force pushing to the branch. - It prevents **anyone** from deleting the branch. diff --git a/doc/workflow/img/repository_mirroring_push_settings.png b/doc/workflow/img/repository_mirroring_push_settings.png Binary files differindex 21a6aca4526..3c0eacaa2df 100644 --- a/doc/workflow/img/repository_mirroring_push_settings.png +++ b/doc/workflow/img/repository_mirroring_push_settings.png diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 6b438235258..9a80671f996 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -17,6 +17,18 @@ module API @project # rubocop:disable Gitlab/ModuleWithInstanceVariables end + def access_checker_for(actor, protocol) + access_checker_klass.new(actor.key_or_user, project, protocol, + authentication_abilities: ssh_authentication_abilities, + namespace_path: namespace_path, + project_path: project_path, + redirected_path: redirected_path) + end + + def access_checker_klass + repo_type.access_checker_class + end + def ssh_authentication_abilities [ :read_project, diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 622032b8355..087985d34b0 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -35,36 +35,14 @@ module API # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList - # rubocop: disable CodeReuse/ActiveRecord post "/allowed" do # Stores some Git-specific env thread-safely env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if project - actor = - if params[:key_id] - Key.find_by(id: params[:key_id]) - elsif params[:user_id] - User.find_by(id: params[:user_id]) - elsif params[:username] - UserFinder.new(params[:username]).find_by_username - end - - protocol = params[:protocol] - - actor.update_last_used_at if actor.is_a?(Key) - user = - if actor.is_a?(Key) - actor.user - else - actor - end - - access_checker_klass = repo_type.access_checker_class - access_checker = access_checker_klass.new(actor, project, - protocol, authentication_abilities: ssh_authentication_abilities, - namespace_path: namespace_path, project_path: project_path, - redirected_path: redirected_path) + actor = Support::GitAccessActor.from_params(params) + actor.update_last_used_at! + access_checker = access_checker_for(actor, params[:protocol]) check_result = begin result = access_checker.check(params[:action], params[:changes]) @@ -78,15 +56,15 @@ module API break response_with_status(code: 404, success: false, message: e.message) end - log_user_activity(actor) + log_user_activity(actor.user) case check_result when ::Gitlab::GitAccessResult::Success payload = { gl_repository: gl_repository, gl_project_path: gl_project_path, - gl_id: Gitlab::GlId.gl_id(user), - gl_username: user&.username, + gl_id: Gitlab::GlId.gl_id(actor.user), + gl_username: actor.username, git_config_options: [], gitaly: gitaly_payload(params[:action]), gl_console_messages: check_result.console_messages @@ -105,7 +83,6 @@ module API response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) end end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord post "/lfs_authenticate" do diff --git a/lib/api/support/git_access_actor.rb b/lib/api/support/git_access_actor.rb new file mode 100644 index 00000000000..2e0962c6295 --- /dev/null +++ b/lib/api/support/git_access_actor.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module API + module Support + class GitAccessActor + attr_reader :user + + def initialize(user: nil, key: nil) + @user = user + @key = key + + @user = key.user if !user && key + end + + def self.from_params(params) + if params[:key_id] + new(key: Key.find_by_id(params[:key_id])) + elsif params[:user_id] + new(user: UserFinder.new(params[:user_id]).find_by_id) + elsif params[:username] + new(user: UserFinder.new(params[:username]).find_by_username) + end + end + + def key_or_user + key || user + end + + def username + user&.username + end + + def update_last_used_at! + key&.update_last_used_at + end + + private + + attr_reader :key + end + end +end diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 0a9a52a73a1..8acd3917d81 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -44,7 +44,7 @@ module Banzai Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled Gitlab.config.asset_proxy['url'] = application_settings.asset_proxy_url Gitlab.config.asset_proxy['secret_key'] = application_settings.asset_proxy_secret_key - Gitlab.config.asset_proxy['whitelist'] = application_settings.asset_proxy_whitelist || [Gitlab.config.gitlab.host] + Gitlab.config.asset_proxy['whitelist'] = determine_whitelist(application_settings) Gitlab.config.asset_proxy['domain_regexp'] = compile_whitelist(Gitlab.config.asset_proxy.whitelist) else Gitlab.config.asset_proxy['enabled'] = ::ApplicationSetting.defaults[:asset_proxy_enabled] @@ -57,6 +57,10 @@ module Banzai escaped = domain_list.map { |domain| Regexp.escape(domain).gsub('\*', '.*?') } Regexp.new("^(#{escaped.join('|')})$", Regexp::IGNORECASE) end + + def self.determine_whitelist(application_settings) + application_settings.asset_proxy_whitelist.presence || [Gitlab.config.gitlab.host] + end end end end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 31dd61ae6cf..97e78ecf094 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -3,7 +3,7 @@ module Gitlab module Auth class UniqueIpsLimiter - USER_UNIQUE_IPS_PREFIX = 'user_unique_ips'.freeze + USER_UNIQUE_IPS_PREFIX = 'user_unique_ips' class << self def limit_user_id!(user_id) diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index bba7e2cbb3c..97755117edc 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -20,7 +20,7 @@ module Gitlab module UserAuthFinders include Gitlab::Utils::StrongMemoize - PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze + PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token # Check the Rails session for valid authentication details diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb index e948cedaad5..de0c357ab1c 100644 --- a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb +++ b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize RESCHEDULE_DELAY = 3.hours - WORKER = 'PopulateMergeRequestAssigneesTable'.freeze + WORKER = 'PopulateMergeRequestAssigneesTable' DeadJobsError = Class.new(StandardError) def perform diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 2ac51dd7b55..3d943205783 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -10,15 +10,15 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize FIND_BATCH_SIZE = 500 - RELATIVE_UPLOAD_DIR = "uploads".freeze + RELATIVE_UPLOAD_DIR = "uploads" ABSOLUTE_UPLOAD_DIR = File.join( Gitlab.config.uploads.storage_path, RELATIVE_UPLOAD_DIR ) - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze - EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze - EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*" + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*" # This class is used to iterate over batches of # `untracked_files_for_uploads` rows. diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index ff2694abd5e..93c6fdcf69c 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -7,7 +7,7 @@ module Gitlab attr_reader :project, :project_key, :repository_slug, :client, :errors, :users attr_accessor :logger - REMOTE_NAME = 'bitbucket_server'.freeze + REMOTE_NAME = 'bitbucket_server' BATCH_SIZE = 100 TempBranch = Struct.new(:name, :sha) diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 67a65d61441..7b013567a03 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -3,8 +3,8 @@ module Gitlab module Checks class LfsCheck < BaseChecker - LOG_MESSAGE = "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...".freeze - ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze + LOG_MESSAGE = 'Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...' + ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' def validate! return unless Feature.enabled?(:lfs_check, default_enabled: true) diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index 0058a402a62..362562068e9 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -3,7 +3,7 @@ module Gitlab module Checks class ProjectCreated < PostPushMessage - PROJECT_CREATED = "project_created".freeze + PROJECT_CREATED = "project_created" def message <<~MESSAGE diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index cb3b7acaaad..6f04fddc6c4 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -3,7 +3,7 @@ module Gitlab module Checks class ProjectMoved < PostPushMessage - REDIRECT_NAMESPACE = "redirect_namespace".freeze + REDIRECT_NAMESPACE = "redirect_namespace" def initialize(project, user, protocol, redirected_path) @redirected_path = redirected_path diff --git a/lib/gitlab/ci/build/port.rb b/lib/gitlab/ci/build/port.rb index 6c4656ffea2..017b31b4f77 100644 --- a/lib/gitlab/ci/build/port.rb +++ b/lib/gitlab/ci/build/port.rb @@ -4,8 +4,8 @@ module Gitlab module Ci module Build class Port - DEFAULT_PORT_NAME = 'default_port'.freeze - DEFAULT_PORT_PROTOCOL = 'http'.freeze + DEFAULT_PORT_NAME = 'default_port' + DEFAULT_PORT_PROTOCOL = 'http' attr_reader :number, :protocol, :name diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 7fcabc035ac..48111ae5717 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -4,9 +4,9 @@ module Gitlab module Ci module Build class Step - WHEN_ON_FAILURE = 'on_failure'.freeze - WHEN_ON_SUCCESS = 'on_success'.freeze - WHEN_ALWAYS = 'always'.freeze + WHEN_ON_FAILURE = 'on_failure' + WHEN_ON_SUCCESS = 'on_success' + WHEN_ALWAYS = 'always' attr_reader :name attr_accessor :script, :timeout, :when, :allow_failure diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 7b94af24c09..ef07c319ce4 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[key untracked paths policy].freeze - DEFAULT_POLICY = 'pull-push'.freeze + DEFAULT_POLICY = 'pull-push' validations do validates :config, allowed_keys: ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 6e11c582750..3009c7e8329 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -15,7 +15,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache dependencies needs before_script after_script variables - environment coverage retry parallel extends].freeze + environment coverage retry parallel extends interruptible].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -37,6 +37,7 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true + validates :interruptible, boolean: true validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } @@ -122,10 +123,11 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, - :parallel, :needs + :parallel, :needs, :interruptible attributes :script, :tags, :allow_failure, :when, :dependencies, - :needs, :retry, :parallel, :extends, :start_in, :rules + :needs, :retry, :parallel, :extends, :start_in, :rules, + :interruptible def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -207,6 +209,7 @@ module Gitlab coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, parallel: parallel_defined? ? parallel_value.to_i : nil, + interruptible: interruptible_defined? ? interruptible_value : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored?, diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 54f4cf74c4d..db56f6a9b00 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -8,7 +8,7 @@ module Gitlab class Template < Base attr_reader :location, :project - SUFFIX = '.gitlab-ci.yml'.freeze + SUFFIX = '.gitlab-ci.yml' def initialize(params, context) @location = params[:template] diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 94f4a4e36c9..1d7e7ea0f9a 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -3,8 +3,8 @@ module Gitlab module Ci class CronParser - VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze - VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' + VALID_SYNTAX_SAMPLE_CRON = '* * * * *' def initialize(cron, cron_timezone = 'UTC') @cron = cron diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 292e273a03a..fdeaad698b9 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -4,10 +4,10 @@ module Gitlab module Ci module Reports class TestCase - STATUS_SUCCESS = 'success'.freeze - STATUS_FAILED = 'failed'.freeze - STATUS_SKIPPED = 'skipped'.freeze - STATUS_ERROR = 'error'.freeze + STATUS_SUCCESS = 'success' + STATUS_FAILED = 'failed' + STATUS_SKIPPED = 'skipped' + STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 5c1c0c142e5..f704266b73d 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -54,7 +54,7 @@ variables: ROLLOUT_RESOURCE_TYPE: deployment - DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 stages: - build @@ -73,16 +73,16 @@ stages: - cleanup include: - - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml # Override DAST job to exclude master branch dast: diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 2e1eab270ff..501d91fa9ad 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -41,6 +41,7 @@ module Gitlab coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), needs_attributes: job[:needs]&.map { |need| { name: need } }, + interruptible: job[:interruptible], options: { image: job[:image], services: job[:services], diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb index 5aace564c2d..3d35d474f5d 100644 --- a/lib/gitlab/cleanup/project_upload_file_finder.rb +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -5,9 +5,9 @@ module Gitlab class ProjectUploadFileFinder FIND_BATCH_SIZE = 500 ABSOLUTE_UPLOAD_DIR = FileUploader.root.freeze - EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*".freeze - EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze - EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*" + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*" + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*" # Paths are relative to the upload directory def each_file_batch(batch_size: FIND_BATCH_SIZE, &block) diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 5424298723e..17ad07bfc0c 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -83,7 +83,8 @@ module Gitlab docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", qa: "~QA", - test: "~test for `spec/features/*`" + test: "~test for `spec/features/*`", + engineering_productivity: "Engineering Productivity for CI config review" }.freeze CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -110,7 +111,8 @@ module Gitlab karma\.config\.js | webpack\.config\.js | package\.json | - yarn\.lock + yarn\.lock | + \.gitlab/ci/frontend\.gitlab-ci\.yml )\z}x => :frontend, %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, @@ -124,7 +126,8 @@ module Gitlab %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, - %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, + %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, + %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, %r{\A\.rubocop(_todo)?\.yml\z} => :backend, diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 2789706aa3b..4ad66f61c2b 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -42,6 +42,8 @@ module Gitlab area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1] area && labels.any?("devops::#{area.downcase}") if kind == :reviewer + when :engineering_productivity + role[/Engineering Productivity/] if kind == :reviewer else capabilities(project).include?("#{kind} #{category}") end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 57a413f8e04..5a42952796c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -459,29 +459,19 @@ module Gitlab check_trigger_permissions!(table) - old_col = column_for(table, old) - new_type = type || old_col.type - - add_column(table, new, new_type, - limit: old_col.limit, - precision: old_col.precision, - scale: old_col.scale) - - # We set the default value _after_ adding the column so we don't end up - # updating any existing data with the default value. This isn't - # necessary since we copy over old values further down. - change_column_default(table, new, old_col.default) unless old_col.default.nil? + create_column_from(table, old, new, type: type) install_rename_triggers(table, old, new) - - update_column_in_batches(table, new, Arel::Table.new(table)[old]) - - change_column_null(table, new, false) unless old_col.null - - copy_indexes(table, old, new) - copy_foreign_keys(table, old, new) end + # Reverses operations performed by rename_column_concurrently. + # + # This method takes care of removing previously installed triggers as well + # as removing the new column. + # + # table - The name of the database table. + # old - The name of the old column. + # new - The name of the new column. def undo_rename_column_concurrently(table, old, new) trigger_name = rename_trigger_name(table, old, new) @@ -557,6 +547,18 @@ module Gitlab remove_column(table, old) end + # Reverses the operations performed by cleanup_concurrent_column_rename. + # + # This method adds back the old_column removed + # by cleanup_concurrent_column_rename. + # It also adds back the (old_column > new_column) trigger that is removed + # by cleanup_concurrent_column_rename. + # + # table - The name of the database table containing the column. + # old - The old column name. + # new - The new column name. + # type - The type of the old column. If no type is given the new column's + # type is used. def undo_cleanup_concurrent_column_rename(table, old, new, type: nil) if transaction_open? raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction' @@ -564,26 +566,9 @@ module Gitlab check_trigger_permissions!(table) - new_column = column_for(table, new) - - add_column(table, old, type || new_column.type, - limit: new_column.limit, - precision: new_column.precision, - scale: new_column.scale) - - # We set the default value _after_ adding the column so we don't end up - # updating any existing data with the default value. This isn't - # necessary since we copy over old values further down. - change_column_default(table, old, new_column.default) unless new_column.default.nil? + create_column_from(table, new, old, type: type) install_rename_triggers(table, old, new) - - update_column_in_batches(table, old, Arel::Table.new(table)[new]) - - change_column_null(table, old, false) unless new_column.null - - copy_indexes(table, new, old) - copy_foreign_keys(table, new, old) end # Changes the column type of a table using a background migration. @@ -1076,6 +1061,28 @@ into similar problems in the future (e.g. when new tables are created). private + def create_column_from(table, old, new, type: nil) + old_col = column_for(table, old) + new_type = type || old_col.type + + add_column(table, new, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new, old_col.default) unless old_col.default.nil? + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + def validate_timestamp_column_name!(column_name) return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index ddbabc9098e..776e80701f1 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -12,7 +12,7 @@ module Gitlab # using them as if they were stored as string values. This gives you the # ease of use of string values, but without the storage overhead. class ShaAttribute < BINARY_TYPE - PACK_FORMAT = 'H*'.freeze + PACK_FORMAT = 'H*' # Casts binary data to a SHA1 in hexadecimal. def deserialize(value) diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index ec38bd769a3..5debb754943 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -5,8 +5,8 @@ module Gitlab class Message attr_reader :path, :offline - OFFLINE = "\e[31moffline\e[0m".freeze - ONLINE = "\e[32monline\e[0m".freeze + OFFLINE = "\e[31moffline\e[0m" + ONLINE = "\e[32monline\e[0m" # path - The file path of the migration. # offline - When set to `true` the migration will require downtime. diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 86e532766b1..20c31e06905 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -1,11 +1,12 @@ +# coding: utf-8 # frozen_string_literal: true # rubocop: disable Rails/Output module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze - CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce' + CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git' CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze PLEASE_READ_THIS_BANNER = %Q{ diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 2395e7be026..1d2f0d7bbf4 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -4,7 +4,7 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - SHARED_STATE_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:' def get(key) Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index df9f33baec2..8d13c74dca2 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -7,11 +7,11 @@ module Gitlab # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' BLANK_SHA = ('0' * 40).freeze COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze - TAG_REF_PREFIX = "refs/tags/".freeze - BRANCH_REF_PREFIX = "refs/heads/".freeze + TAG_REF_PREFIX = "refs/tags/" + BRANCH_REF_PREFIX = "refs/heads/" BaseError = Class.new(StandardError) CommandError = Class.new(BaseError) diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb index b7019a221ac..efd84e30ad9 100644 --- a/lib/gitlab/git/lfs_pointer_file.rb +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -3,8 +3,8 @@ module Gitlab module Git class LfsPointerFile - VERSION = "https://git-lfs.github.com/spec/v1".freeze - VERSION_LINE = "version #{VERSION}".freeze + VERSION = "https://git-lfs.github.com/spec/v1" + VERSION_LINE = "version #{VERSION}" def initialize(data) @data = data diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 27032602828..4ea618f063b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -15,9 +15,9 @@ module Gitlab SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 - GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git' GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout - EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000' NoRepository = Class.new(StandardError) InvalidRepository = Class.new(StandardError) diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 03c2c1367b0..cf8571bf917 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -5,7 +5,7 @@ module Gitlab module Git module Util - LINE_SEP = "\n".freeze + LINE_SEP = "\n" def self.count_lines(string) case string[-1] diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 211915f1d87..c81603a1aa9 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :project, :object # The base cache key to use for storing/retrieving issuable IDs. - CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'.freeze + CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}' # project - An instance of `Project`. # object - The object to look up or set a database ID for. diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index d2479a8f565..cad39e48e43 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :project # The base cache key to use for storing/retrieving label IDs. - CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'.freeze + CACHE_KEY = 'github-import/label-finder/%{project}/%{name}' # project - An instance of `Project`. def initialize(project) diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index 5625730e796..a157a1e1ff5 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :project # The base cache key to use for storing/retrieving milestone IDs. - CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'.freeze + CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}' # project - An instance of `Project` def initialize(project) diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index c3db2d0b469..a3e7b3c1afc 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :cache_key # The base cache key to use for storing the last page number. - CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}'.freeze + CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}' def initialize(project, collection) @cache_key = CACHE_KEY % { project: project.id, collection: collection } diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index d4d1357f5a3..849a66d47ed 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -7,7 +7,7 @@ module Gitlab # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = - 'github-importer/already-imported/%{project}/%{collection}'.freeze + 'github-importer/already-imported/%{project}/%{collection}' # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 30283f147ef..51a532437bd 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -16,17 +16,17 @@ module Gitlab # The base cache key to use for caching user IDs for a given GitHub user # ID. - ID_CACHE_KEY = 'github-import/user-finder/user-id/%s'.freeze + ID_CACHE_KEY = 'github-import/user-finder/user-id/%s' # The base cache key to use for caching user IDs for a given GitHub email # address. ID_FOR_EMAIL_CACHE_KEY = - 'github-import/user-finder/id-for-email/%s'.freeze + 'github-import/user-finder/id-for-email/%s' # The base cache key to use for caching the Email addresses of GitHub # usernames. EMAIL_FOR_USERNAME_CACHE_KEY = - 'github-import/user-finder/email-for-username/%s'.freeze + 'github-import/user-finder/email-for-username/%s' # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 0b11ea2f608..c7f430490d6 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -74,9 +74,9 @@ module Gitlab # Authorizing fields representing scalars, or a simple field with an object resolved_type if allowed_access?(current_user, authorizing_object) elsif @field.connection? - # A connection with pagination, modify the visible nodes in on the + # A connection with pagination, modify the visible nodes on the # connection type in place - resolved_type.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) } + resolved_type.object.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) } resolved_type elsif resolved_type.is_a? Array # A simple list of rendered types each object being an object to authorize diff --git a/lib/gitlab/graphql/representation/submodule_tree_entry.rb b/lib/gitlab/graphql/representation/submodule_tree_entry.rb index 65716dff75d..8d17cb9eecc 100644 --- a/lib/gitlab/graphql/representation/submodule_tree_entry.rb +++ b/lib/gitlab/graphql/representation/submodule_tree_entry.rb @@ -4,6 +4,8 @@ module Gitlab module Graphql module Representation class SubmoduleTreeEntry < SimpleDelegator + include GlobalID::Identification + class << self def decorate(submodules, tree) repository = tree.repository diff --git a/lib/gitlab/graphql/representation/tree_entry.rb b/lib/gitlab/graphql/representation/tree_entry.rb index 7ea83db5876..7e347081a9b 100644 --- a/lib/gitlab/graphql/representation/tree_entry.rb +++ b/lib/gitlab/graphql/representation/tree_entry.rb @@ -4,6 +4,8 @@ module Gitlab module Graphql module Representation class TreeEntry < SimpleDelegator + include GlobalID::Identification + class << self def decorate(entries, repository) return if entries.nil? diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index 898733fea5d..e560f87bf98 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -5,7 +5,7 @@ module Gitlab class GitalyCheck extend BaseAbstractCheck - METRIC_PREFIX = 'gitaly_health_check'.freeze + METRIC_PREFIX = 'gitaly_health_check' class << self def readiness diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb46bd657e8..d08848a65a8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -7,7 +7,7 @@ module Gitlab # For every version update the version history in these docs must be kept up to date: # - development/import_export.md # - user/project/settings/import_export.md - VERSION = '0.2.4'.freeze + VERSION = '0.2.4' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index d39b6fe5955..c5fb39b7b52 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -10,7 +10,7 @@ module Gitlab StrategyError = Class.new(StandardError) - AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action' private diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index acb7f225b17..aaa70f0b36d 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -4,9 +4,9 @@ module Gitlab module ImportExport module AfterExportStrategies class WebUploadStrategy < BaseAfterExportStrategy - PUT_METHOD = 'PUT'.freeze - POST_METHOD = 'POST'.freeze - INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + PUT_METHOD = 'PUT' + POST_METHOD = 'POST' + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.' validates :url, addressable_url: true diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 8b346f6d7d2..eece4edf895 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -2,9 +2,9 @@ module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '-unsubscribe'.freeze - UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe'.freeze - WILDCARD_PLACEHOLDER = '%{key}'.freeze + UNSUBSCRIBE_SUFFIX = '-unsubscribe' + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe' + WILDCARD_PLACEHOLDER = '%{key}' class << self def enabled? diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e97e961771c..90dbe4d005d 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -17,7 +17,7 @@ module Gitlab # push to that array when done. Once the waiter has popped `count` items, it # knows all the jobs are done. class JobWaiter - KEY_PREFIX = "gitlab:job_waiter".freeze + KEY_PREFIX = "gitlab:job_waiter" def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 6e4286589c1..16ed0cb0f8e 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,12 +3,12 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.14.3'.freeze - KUBECTL_VERSION = '1.11.10'.freeze - NAMESPACE = 'gitlab-managed-apps'.freeze - SERVICE_ACCOUNT = 'tiller'.freeze - CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze - CLUSTER_ROLE = 'cluster-admin'.freeze + HELM_VERSION = '2.14.3' + KUBECTL_VERSION = '1.11.10' + NAMESPACE = 'gitlab-managed-apps' + SERVICE_ACCOUNT = 'tiller' + CLUSTER_ROLE_BINDING = 'tiller-admin' + CLUSTER_ROLE = 'cluster-admin' end end end diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb index 81317e532b2..d247662dc3b 100644 --- a/lib/gitlab/kubernetes/pod.rb +++ b/lib/gitlab/kubernetes/pod.rb @@ -3,11 +3,11 @@ module Gitlab module Kubernetes module Pod - PENDING = 'Pending'.freeze - RUNNING = 'Running'.freeze - SUCCEEDED = 'Succeeded'.freeze - FAILED = 'Failed'.freeze - UNKNOWN = 'Unknown'.freeze + PENDING = 'Pending' + RUNNING = 'Running' + SUCCEEDED = 'Succeeded' + FAILED = 'Failed' + UNKNOWN = 'Unknown' PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 128a5dd8936..2ec8c268d09 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -40,7 +40,7 @@ module Gitlab end def self.cache_key - 'logger:'.freeze + self.full_log_path.to_s + 'logger:' + self.full_log_path.to_s end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index c068f8017fd..2ed5878286a 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -15,7 +15,7 @@ module Gitlab attach_to :action_view - SERIES = 'views'.freeze + SERIES = 'views' def render_template(event) track(event) if current_transaction diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index d7986685c65..ba2a0b2ecf8 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -14,7 +14,7 @@ module Gitlab THREAD_KEY = :_gitlab_metrics_transaction # The series to store events (e.g. Git pushes) in. - EVENT_SERIES = 'events'.freeze + EVENT_SERIES = 'events' attr_reader :tags, :values, :method, :metrics @@ -78,7 +78,7 @@ module Gitlab # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) filtered_tags = filter_tags(tags) - self.class.transaction_metric(event_name, :counter, prefix: 'event_', use_feature_flag: true, tags: filtered_tags).increment(filtered_tags.merge(labels)) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels)) @metrics << Metric.new(EVENT_SERIES, { count: 1 }, filtered_tags.merge(event: event_name), :event) end @@ -155,12 +155,11 @@ module Gitlab with_feature :prometheus_metrics_transaction_allocated_memory end - def self.transaction_metric(name, type, prefix: nil, use_feature_flag: false, tags: {}) + def self.transaction_metric(name, type, prefix: nil, tags: {}) metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym fetch_metric(type, metric_name) do docstring "Transaction #{prefix}#{name} #{type}" base_labels tags.merge(BASE_LABELS) - with_feature "prometheus_transaction_#{prefix}#{name}_total".to_sym if use_feature_flag if type == :gauge multiprocess_mode :livesum diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index b2a43d46fb2..fa17548723e 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -3,8 +3,8 @@ module Gitlab module Metrics class WebTransaction < Transaction - CONTROLLER_KEY = 'action_controller.instance'.freeze - ENDPOINT_KEY = 'api.endpoint'.freeze + CONTROLLER_KEY = 'action_controller.instance' + ENDPOINT_KEY = 'api.endpoint' ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) def initialize(env) diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 433151b80e7..86b28e4e20a 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -28,7 +28,7 @@ module Gitlab module Middleware class Multipart - RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'.freeze + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' class Handler def initialize(env, message) diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 53e55269c6e..802ff18fc58 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -5,9 +5,9 @@ module Gitlab class ReadOnly class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON = 'application/json' APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze - ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' WHITELISTED_GIT_ROUTES = { 'projects/git_http' => %w{git_upload_pack git_receive_pack} diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index d9c28ff1181..ee2ef91c65c 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -125,9 +125,9 @@ module Gitlab # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_START_CHAR = '[a-zA-Z0-9_\.]'.freeze - PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]*'.freeze - NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + PATH_START_CHAR = '[a-zA-Z0-9_\.]' + PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]*' + NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]' NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 68af290d069..4445c876e7a 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -2,7 +2,7 @@ module Gitlab module PerformanceBar - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2' EXPIRY_TIME_L1_CACHE = 1.minute EXPIRY_TIME_L2_CACHE = 5.minutes diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index 0f69990df63..fd66787b029 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -2,7 +2,7 @@ module Gitlab class PollingInterval - HEADER_NAME = 'Poll-Interval'.freeze + HEADER_NAME = 'Poll-Interval' def self.set_header(response, interval:) if polling_enabled? diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb index 536fc9dae3a..886c2c32d36 100644 --- a/lib/gitlab/private_commit_email.rb +++ b/lib/gitlab/private_commit_email.rb @@ -2,7 +2,7 @@ module Gitlab module PrivateCommitEmail - TOKEN = "_private".freeze + TOKEN = "_private" class << self def regex diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 425c30d67fe..3f26b84be20 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -3,7 +3,7 @@ module Gitlab module Profiler - FILTERED_STRING = '[FILTERED]'.freeze + FILTERED_STRING = '[FILTERED]' IGNORE_BACKTRACES = %w[ lib/gitlab/i18n.rb diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index bd4ca578840..ee3d98f3602 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -3,7 +3,7 @@ module Gitlab module Prometheus module AdditionalMetricsParser - CONFIG_ROOT = 'config/prometheus'.freeze + CONFIG_ROOT = 'config/prometheus' MUTEX = Mutex.new extend self diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 682edfc4259..a2296d265cd 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -7,10 +7,12 @@ module Gitlab keys: [ :create, :description, + :label, :merge_when_pipeline_succeeds, :remove_source_branch, :target, - :title + :title, + :unlabel ] }, ci: { @@ -18,6 +20,11 @@ module Gitlab } }).freeze + MULTI_VALUE_OPTIONS = [ + %w[merge_request label], + %w[merge_request unlabel] + ].freeze + NAMESPACE_ALIASES = HashWithIndifferentAccess.new({ mr: :merge_request }).freeze @@ -50,12 +57,22 @@ module Gitlab next if [namespace, key].any?(&:nil?) options[namespace] ||= HashWithIndifferentAccess.new - options[namespace][key] = value + + if option_multi_value?(namespace, key) + options[namespace][key] ||= HashWithIndifferentAccess.new(0) + options[namespace][key][value] += 1 + else + options[namespace][key] = value + end end options end + def option_multi_value?(namespace, key) + MULTI_VALUE_OPTIONS.any? { |arr| arr == [namespace, key] } + end + def parse_option(option) parts = OPTION_MATCHER.match(option) return unless parts diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb index 949ae79a047..f6ffbfe2645 100644 --- a/lib/gitlab/query_limiting/middleware.rb +++ b/lib/gitlab/query_limiting/middleware.rb @@ -5,8 +5,8 @@ module Gitlab # Middleware for reporting (or raising) when a request performs more than a # certain amount of database queries. class Middleware - CONTROLLER_KEY = 'action_controller.instance'.freeze - ENDPOINT_KEY = 'api.endpoint'.freeze + CONTROLLER_KEY = 'action_controller.instance' + ENDPOINT_KEY = 'api.endpoint' def initialize(app) @app = app diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index e5d99ebee35..5cf24823ef5 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -1,3 +1,4 @@ +# coding: utf-8 # frozen_string_literal: true module Gitlab @@ -6,8 +7,8 @@ module Gitlab extend ActiveSupport::Concern include Gitlab::QuickActions::Dsl - SHRUG = '¯\\＿(ツ)＿/¯'.freeze - TABLEFLIP = '(╯°□°)╯︵ ┻━┻'.freeze + SHRUG = '¯\\＿(ツ)＿/¯' + TABLEFLIP = '(╯°□°)╯︵ ┻━┻' included do # Issue, MergeRequest, Epic: quick actions definitions diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index da28fbf5be0..869627ac585 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -25,7 +25,11 @@ module Gitlab Chronic.parse(due_date_param).try(:to_date) end command :due do |due_date| - @updates[:due_date] = due_date if due_date + if due_date + @updates[:due_date] = due_date + else + @execution_message[:due] = _('Failed to set due date because the date format is invalid.') + end end desc _('Remove due date') diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 6e8403ad878..6e31c506438 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -6,9 +6,9 @@ require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? module Gitlab module Redis class Cache < ::Gitlab::Redis::Wrapper - CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze - REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + CACHE_NAMESPACE = 'cache:gitlab' + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380' + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE' class << self def default_url diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 8b42c269dd0..0375e4a221a 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -6,10 +6,10 @@ require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class Queues < ::Gitlab::Redis::Wrapper - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze - REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + SIDEKIQ_NAMESPACE = 'resque:gitlab' + MAILROOM_NAMESPACE = 'mail_room:gitlab' + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381' + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE' class << self def default_url diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 270a19e780c..35356083f26 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -6,12 +6,12 @@ require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper - SESSION_NAMESPACE = 'session:gitlab'.freeze - USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze - USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze - IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab'.freeze - DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + SESSION_NAMESPACE = 'session:gitlab' + USER_SESSIONS_NAMESPACE = 'session:user:gitlab' + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab' + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382' + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE' class << self def default_url diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 07a1e20b076..fa1615a5953 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,8 +8,8 @@ require 'active_support/core_ext/module/delegation' module Gitlab module Redis class Wrapper - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + DEFAULT_REDIS_URL = 'redis://localhost:6379' + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE' class << self delegate :params, :url, to: :new diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 033e451dbee..dd1482da40d 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -4,7 +4,7 @@ require 'fileutils' module Gitlab module RequestProfiler - PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" def all Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb index 6612347438e..eeb0cc75ef9 100644 --- a/lib/gitlab/shard_health_cache.rb +++ b/lib/gitlab/shard_health_cache.rb @@ -2,7 +2,7 @@ module Gitlab class ShardHealthCache - HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze + HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards' HEALTHY_SHARDS_TIMEOUT = 300 # Clears the Redis set storing the list of healthy shards diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb index 9842f1f53f7..a58b33534bf 100644 --- a/lib/gitlab/sidekiq_monitor.rb +++ b/lib/gitlab/sidekiq_monitor.rb @@ -4,7 +4,7 @@ module Gitlab class SidekiqMonitor < Daemon include ::Gitlab::Utils::StrongMemoize - NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze + NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications' CANCEL_DEADLINE = 24.hours.seconds RECONNECT_TIME = 3.seconds diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 0f890a12134..0dafccb3d34 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -18,7 +18,7 @@ module Gitlab # expire after a certain period of time to prevent storing too many keys in # Redis. module SidekiqStatus - STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + STATUS_KEY = 'gitlab-sidekiq-status:%s' # The default time (in seconds) after which a status key is expired # automatically. The default of 30 minutes should be more than sufficient diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9c35d200dcb..fab504aa603 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -49,7 +49,7 @@ module Gitlab hostname = uri.hostname port = get_port(uri) - address_info = get_address_info(hostname, port) + address_info = get_address_info(hostname, port, dns_rebind_protection) return [uri, nil] unless address_info ip_address = ip_address(address_info) @@ -110,11 +110,15 @@ module Gitlab validate_unicode_restriction(uri) if ascii_only end - def get_address_info(hostname, port) + def get_address_info(hostname, port, dns_rebind_protection) Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError + # If the dns rebinding protection is not enabled, we allow + # urls that can't be resolved at this point. + return unless dns_rebind_protection + # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS diff --git a/lib/gitlab/url_helpers.rb b/lib/gitlab/url_helpers.rb index 883585c196f..2a7919883c5 100644 --- a/lib/gitlab/url_helpers.rb +++ b/lib/gitlab/url_helpers.rb @@ -2,7 +2,7 @@ module Gitlab class UrlHelpers - WSS_PROTOCOL = "wss".freeze + WSS_PROTOCOL = "wss" def self.as_wss(url) return unless url.present? diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 139ec6e384a..db67e4fd479 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -7,13 +7,13 @@ require 'uri' module Gitlab class Workhorse - SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'.freeze - VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze - INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze - NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze + SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' + VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' + NOTIFICATION_CHANNEL = 'workhorse:notifications' ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze - DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type'.freeze + DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type' include JwtAuthenticatable diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 629daccf006..d26ce9fa911 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4984,6 +4984,9 @@ msgstr "" msgid "Failed to save preferences." msgstr "" +msgid "Failed to set due date because the date format is invalid." +msgstr "" + msgid "Failed to update branch!" msgstr "" @@ -7206,7 +7209,7 @@ msgstr "" msgid "Monday" msgstr "" -msgid "Monitor your errors by integrating with Sentry" +msgid "Monitor your errors by integrating with Sentry." msgstr "" msgid "Monitoring" @@ -13020,6 +13023,9 @@ msgid_plural "When these merge requests are accepted" msgstr[0] "" msgstr[1] "" +msgid "When using the <code>http://</code> or <code>https://</code> protocols, please provide the exact URL to the repository. HTTP redirects will not be followed." +msgstr "" + msgid "When:" msgstr "" diff --git a/plugins/.gitignore b/plugins/.gitignore new file mode 100644 index 00000000000..e4ccdc9e2ec --- /dev/null +++ b/plugins/.gitignore @@ -0,0 +1,5 @@ +* +!*/ +!.gitignore +!.gitkeep +!examples/* diff --git a/shared/.gitignore b/shared/.gitignore new file mode 100644 index 00000000000..5c0276adafd --- /dev/null +++ b/shared/.gitignore @@ -0,0 +1,4 @@ +* +!*/ +!.gitignore +!.gitkeep diff --git a/spec/features/help_pages_spec.rb b/spec/features/help_pages_spec.rb index bcd2b90d3bb..88a7aa51326 100644 --- a/spec/features/help_pages_spec.rb +++ b/spec/features/help_pages_spec.rb @@ -58,7 +58,7 @@ describe 'Help Pages' do before do stub_application_setting(version_check_enabled: true) - allow(Rails.env).to receive(:production?).and_return(true) + stub_rails_env('production') allow(VersionCheck).to receive(:url).and_return('/version-check-url') sign_in(create(:user)) diff --git a/spec/features/user_can_display_performance_bar_spec.rb b/spec/features/user_can_display_performance_bar_spec.rb index 154e948015f..8b3f193f418 100644 --- a/spec/features/user_can_display_performance_bar_spec.rb +++ b/spec/features/user_can_display_performance_bar_spec.rb @@ -37,7 +37,7 @@ describe 'User can display performance bar', :js do shared_examples 'performance bar is enabled by default in development' do before do - allow(Rails.env).to receive(:development?).and_return(true) + stub_rails_env('development') end it 'shows the performance bar by default' do diff --git a/spec/frontend/clusters/components/applications_spec.js b/spec/frontend/clusters/components/applications_spec.js index 221ebb143be..1d8984cea0a 100644 --- a/spec/frontend/clusters/components/applications_spec.js +++ b/spec/frontend/clusters/components/applications_spec.js @@ -85,7 +85,44 @@ describe('Applications', () => { }); it('renders a row for Jupyter', () => { - expect(vm.$el.querySelector('.js-cluster-application-row-jupyter')).toBeNull(); + expect(vm.$el.querySelector('.js-cluster-application-row-jupyter')).not.toBeNull(); + }); + + it('renders a row for Knative', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-knative')).toBeNull(); + }); + }); + + describe('Instance cluster applications', () => { + beforeEach(() => { + vm = mountComponent(Applications, { + type: CLUSTER_TYPE.INSTANCE, + applications: APPLICATIONS_MOCK_STATE, + }); + }); + + it('renders a row for Helm Tiller', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-helm')).not.toBeNull(); + }); + + it('renders a row for Ingress', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-ingress')).not.toBeNull(); + }); + + it('renders a row for Cert-Manager', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-cert_manager')).not.toBeNull(); + }); + + it('renders a row for Prometheus', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-prometheus')).not.toBeNull(); + }); + + it('renders a row for GitLab Runner', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-runner')).not.toBeNull(); + }); + + it('renders a row for Jupyter', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-jupyter')).not.toBeNull(); }); it('renders a row for Knative', () => { diff --git a/spec/frontend/error_tracking/components/error_tracking_list_spec.js b/spec/frontend/error_tracking/components/error_tracking_list_spec.js index 67e5dc399ac..ce8b8908026 100644 --- a/spec/frontend/error_tracking/components/error_tracking_list_spec.js +++ b/spec/frontend/error_tracking/components/error_tracking_list_spec.js @@ -11,19 +11,24 @@ describe('ErrorTrackingList', () => { let wrapper; let actions; - function mountComponent({ errorTrackingEnabled = true } = {}) { + function mountComponent({ + errorTrackingEnabled = true, + userCanEnableErrorTracking = true, + stubs = { + 'gl-link': GlLink, + }, + } = {}) { wrapper = shallowMount(ErrorTrackingList, { localVue, store, propsData: { indexPath: '/path', enableErrorTrackingLink: '/link', + userCanEnableErrorTracking, errorTrackingEnabled, illustrationPath: 'illustration/path', }, - stubs: { - 'gl-link': GlLink, - }, + stubs, }); } @@ -115,4 +120,23 @@ describe('ErrorTrackingList', () => { expect(wrapper.find(GlButton).exists()).toBeFalsy(); }); }); + + describe('When error tracking is disabled and user is not allowed to enable it', () => { + beforeEach(() => { + mountComponent({ + errorTrackingEnabled: false, + userCanEnableErrorTracking: false, + stubs: { + 'gl-link': GlLink, + 'gl-empty-state': GlEmptyState, + }, + }); + }); + + it('shows empty state', () => { + expect(wrapper.find('a').attributes('href')).toBe( + '/help/user/project/operations/error_tracking.html', + ); + }); + }); }); diff --git a/spec/frontend/ide/components/preview/clientside_spec.js b/spec/frontend/ide/components/preview/clientside_spec.js new file mode 100644 index 00000000000..dfc76628d0c --- /dev/null +++ b/spec/frontend/ide/components/preview/clientside_spec.js @@ -0,0 +1,318 @@ +import Vuex from 'vuex'; +import { GlLoadingIcon } from '@gitlab/ui'; +import { shallowMount, createLocalVue } from '@vue/test-utils'; +import smooshpack from 'smooshpack'; +import Clientside from '~/ide/components/preview/clientside.vue'; + +jest.mock('smooshpack', () => ({ + Manager: jest.fn(), +})); + +const localVue = createLocalVue(); +localVue.use(Vuex); + +const dummyPackageJson = () => ({ + raw: JSON.stringify({ + main: 'index.js', + }), +}); + +describe('IDE clientside preview', () => { + let wrapper; + let store; + const storeActions = { + getFileData: jest.fn().mockReturnValue(Promise.resolve({})), + getRawFileData: jest.fn().mockReturnValue(Promise.resolve('')), + }; + + const waitForCalls = () => new Promise(setImmediate); + + const createComponent = ({ state, getters } = {}) => { + store = new Vuex.Store({ + state: { + entries: {}, + links: {}, + ...state, + }, + getters: { + packageJson: () => '', + currentProject: () => ({ + visibility: 'public', + }), + ...getters, + }, + actions: storeActions, + }); + + wrapper = shallowMount(Clientside, { + sync: false, + store, + localVue, + }); + }; + + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + wrapper.destroy(); + }); + + describe('without main entry', () => { + it('creates sandpack manager', () => { + createComponent(); + expect(smooshpack.Manager).not.toHaveBeenCalled(); + }); + }); + describe('with main entry', () => { + beforeEach(() => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + return wrapper.vm.initPreview(); + }); + + it('creates sandpack manager', () => { + expect(smooshpack.Manager).toHaveBeenCalledWith( + '#ide-preview', + { + files: {}, + entry: '/index.js', + showOpenInCodeSandbox: true, + }, + { + fileResolver: { + isFile: expect.any(Function), + readFile: expect.any(Function), + }, + }, + ); + }); + }); + + describe('computed', () => { + describe('normalizedEntries', () => { + it('returns flattened list of blobs with content', () => { + createComponent({ + state: { + entries: { + 'index.js': { type: 'blob', raw: 'test' }, + 'index2.js': { type: 'blob', content: 'content' }, + tree: { type: 'tree' }, + empty: { type: 'blob' }, + }, + }, + }); + + expect(wrapper.vm.normalizedEntries).toEqual({ + '/index.js': { + code: 'test', + }, + '/index2.js': { + code: 'content', + }, + }); + }); + }); + + describe('mainEntry', () => { + it('returns false when package.json is empty', () => { + createComponent(); + expect(wrapper.vm.mainEntry).toBe(false); + }); + + it('returns main key from package.json', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + expect(wrapper.vm.mainEntry).toBe('index.js'); + }); + }); + + describe('showPreview', () => { + it('returns false if no mainEntry', () => { + createComponent(); + expect(wrapper.vm.showPreview).toBe(false); + }); + + it('returns false if loading and mainEntry exists', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + wrapper.setData({ loading: true }); + + expect(wrapper.vm.showPreview).toBe(false); + }); + + it('returns true if not loading and mainEntry exists', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + wrapper.setData({ loading: false }); + + expect(wrapper.vm.showPreview).toBe(true); + }); + }); + + describe('showEmptyState', () => { + it('returns true if no mainEntry exists', () => { + createComponent(); + wrapper.setData({ loading: false }); + expect(wrapper.vm.showEmptyState).toBe(true); + }); + + it('returns false if loading', () => { + createComponent(); + wrapper.setData({ loading: true }); + + expect(wrapper.vm.showEmptyState).toBe(false); + }); + + it('returns false if not loading and mainEntry exists', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + wrapper.setData({ loading: false }); + + expect(wrapper.vm.showEmptyState).toBe(false); + }); + }); + + describe('showOpenInCodeSandbox', () => { + it('returns true when visiblity is public', () => { + createComponent({ getters: { currentProject: () => ({ visibility: 'public' }) } }); + + expect(wrapper.vm.showOpenInCodeSandbox).toBe(true); + }); + + it('returns false when visiblity is private', () => { + createComponent({ getters: { currentProject: () => ({ visibility: 'private' }) } }); + + expect(wrapper.vm.showOpenInCodeSandbox).toBe(false); + }); + }); + + describe('sandboxOpts', () => { + beforeEach(() => { + createComponent({ + state: { + entries: { + 'index.js': { type: 'blob', raw: 'test' }, + 'package.json': dummyPackageJson(), + }, + }, + getters: { + packageJson: dummyPackageJson, + }, + }); + }); + + it('returns sandbox options', () => { + expect(wrapper.vm.sandboxOpts).toEqual({ + files: { + '/index.js': { + code: 'test', + }, + '/package.json': { + code: '{"main":"index.js"}', + }, + }, + entry: '/index.js', + showOpenInCodeSandbox: true, + }); + }); + }); + }); + + describe('methods', () => { + describe('loadFileContent', () => { + beforeEach(() => { + createComponent(); + return wrapper.vm.loadFileContent('package.json'); + }); + + it('calls getFileData', () => { + expect(storeActions.getFileData).toHaveBeenCalledWith( + expect.any(Object), + { + path: 'package.json', + makeFileActive: false, + }, + undefined, // vuex callback + ); + }); + + it('calls getRawFileData', () => { + expect(storeActions.getRawFileData).toHaveBeenCalledWith( + expect.any(Object), + { + path: 'package.json', + }, + undefined, // vuex callback + ); + }); + }); + + describe('update', () => { + beforeEach(() => { + createComponent(); + wrapper.setData({ sandpackReady: true }); + }); + + it('initializes manager if manager is empty', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + wrapper.setData({ sandpackReady: true }); + wrapper.vm.update(); + + jest.advanceTimersByTime(250); + + return waitForCalls().then(() => { + expect(smooshpack.Manager).toHaveBeenCalled(); + }); + }); + + it('calls updatePreview', () => { + wrapper.setData({ + manager: { + listener: jest.fn(), + updatePreview: jest.fn(), + }, + }); + wrapper.vm.update(); + + jest.advanceTimersByTime(250); + expect(wrapper.vm.manager.updatePreview).toHaveBeenCalledWith(wrapper.vm.sandboxOpts); + }); + }); + }); + + describe('template', () => { + it('renders ide-preview element when showPreview is true', () => { + createComponent({ getters: { packageJson: dummyPackageJson } }); + wrapper.setData({ loading: false }); + + return wrapper.vm.$nextTick(() => { + expect(wrapper.find('#ide-preview').exists()).toBe(true); + }); + }); + + it('renders empty state', () => { + createComponent(); + wrapper.setData({ loading: false }); + + return wrapper.vm.$nextTick(() => { + expect(wrapper.text()).toContain( + 'Preview your web application using Web IDE client-side evaluation.', + ); + }); + }); + + it('renders loading icon', () => { + createComponent(); + wrapper.setData({ loading: true }); + + return wrapper.vm.$nextTick(() => { + expect(wrapper.find(GlLoadingIcon).exists()).toBe(true); + }); + }); + }); +}); diff --git a/spec/frontend/lib/utils/axios_utils_spec.js b/spec/frontend/lib/utils/axios_utils_spec.js new file mode 100644 index 00000000000..d5c39567f06 --- /dev/null +++ b/spec/frontend/lib/utils/axios_utils_spec.js @@ -0,0 +1,45 @@ +/* eslint-disable promise/catch-or-return */ + +import AxiosMockAdapter from 'axios-mock-adapter'; + +import axios from '~/lib/utils/axios_utils'; + +describe('axios_utils', () => { + let mock; + + beforeEach(() => { + mock = new AxiosMockAdapter(axios); + mock.onAny('/ok').reply(200); + mock.onAny('/err').reply(500); + expect(axios.countActiveRequests()).toBe(0); + }); + + afterEach(() => axios.waitForAll().finally(() => mock.restore())); + + describe('waitForAll', () => { + it('resolves if there are no requests', () => axios.waitForAll()); + + it('waits for all requests to finish', () => { + const handler = jest.fn(); + axios.get('/ok').then(handler); + axios.get('/err').catch(handler); + + return axios.waitForAll().finally(() => { + expect(handler).toHaveBeenCalledTimes(2); + expect(handler.mock.calls[0][0].status).toBe(200); + expect(handler.mock.calls[1][0].response.status).toBe(500); + }); + }); + }); + + describe('waitFor', () => { + it('waits for requests on a specific URL', () => { + const handler = jest.fn(); + axios.get('/ok').finally(handler); + axios.waitFor('/err').finally(() => { + throw new Error('waitFor on /err should not be called'); + }); + return axios.waitFor('/ok'); + }); + }); +}); diff --git a/spec/frontend/mocks/ce/lib/utils/axios_utils.js b/spec/frontend/mocks/ce/lib/utils/axios_utils.js index a3783b91f95..85fad231d28 100644 --- a/spec/frontend/mocks/ce/lib/utils/axios_utils.js +++ b/spec/frontend/mocks/ce/lib/utils/axios_utils.js @@ -1,3 +1,5 @@ +import EventEmitter from 'events'; + const axios = jest.requireActual('~/lib/utils/axios_utils').default; axios.isMock = true; @@ -13,4 +15,64 @@ axios.defaults.adapter = config => { throw error; }; +// Count active requests and provide a way to wait for them +let activeRequests = 0; +const events = new EventEmitter(); +const onRequest = () => { + activeRequests += 1; +}; + +// Use setImmediate to alloow the response interceptor to finish +const onResponse = config => { + activeRequests -= 1; + setImmediate(() => { + events.emit('response', config); + }); +}; + +const subscribeToResponse = (predicate = () => true) => + new Promise(resolve => { + const listener = (config = {}) => { + if (predicate(config)) { + events.off('response', listener); + resolve(config); + } + }; + + events.on('response', listener); + + // If a request has been made synchronously, setImmediate waits for it to be + // processed and the counter incremented. + setImmediate(listener); + }); + +/** + * Registers a callback function to be run after a request to the given URL finishes. + */ +axios.waitFor = url => subscribeToResponse(({ url: configUrl }) => configUrl === url); + +/** + * Registers a callback function to be run after all requests have finished. If there are no requests waiting, the callback is executed immediately. + */ +axios.waitForAll = () => subscribeToResponse(() => activeRequests === 0); + +axios.countActiveRequests = () => activeRequests; + +axios.interceptors.request.use(config => { + onRequest(); + return config; +}); + +// Remove the global counter +axios.interceptors.response.use( + response => { + onResponse(response.config); + return response; + }, + err => { + onResponse(err.config); + return Promise.reject(err); + }, +); + export default axios; diff --git a/spec/frontend/notes/old_notes_spec.js b/spec/frontend/notes/old_notes_spec.js index b57041cf4d1..96133c601aa 100644 --- a/spec/frontend/notes/old_notes_spec.js +++ b/spec/frontend/notes/old_notes_spec.js @@ -49,17 +49,12 @@ describe('Old Notes (~/notes.js)', () => { setTestTimeoutOnce(4000); }); - afterEach(done => { + afterEach(() => { // The Notes component sets a polling interval. Clear it after every run. // Make sure to use jest.runOnlyPendingTimers() instead of runAllTimers(). jest.clearAllTimers(); - setImmediate(() => { - // Wait for any requests to resolve, otherwise we get failures about - // unmocked requests. - mockAxios.restore(); - done(); - }); + return axios.waitForAll().finally(() => mockAxios.restore()); }); it('loads the Notes class into the DOM', () => { diff --git a/spec/graphql/features/authorization_spec.rb b/spec/graphql/features/authorization_spec.rb index c427893f9cc..9a60ff3b78c 100644 --- a/spec/graphql/features/authorization_spec.rb +++ b/spec/graphql/features/authorization_spec.rb @@ -8,10 +8,10 @@ describe 'Gitlab::Graphql::Authorization' do let(:permission_single) { :foo } let(:permission_collection) { [:foo, :bar] } let(:test_object) { double(name: 'My name') } - let(:query_string) { '{ object() { name } }' } + let(:query_string) { '{ item() { name } }' } let(:result) { execute_query(query_type)['data'] } - subject { result['object'] } + subject { result['item'] } shared_examples 'authorization with a single permission' do it 'returns the protected field when user has permission' do @@ -54,7 +54,7 @@ describe 'Gitlab::Graphql::Authorization' do describe 'with a single permission' do let(:query_type) do query_factory do |query| - query.field :object, type, null: true, resolve: ->(obj, args, ctx) { test_object }, authorize: permission_single + query.field :item, type, null: true, resolve: ->(obj, args, ctx) { test_object }, authorize: permission_single end end @@ -65,7 +65,7 @@ describe 'Gitlab::Graphql::Authorization' do let(:query_type) do permissions = permission_collection query_factory do |qt| - qt.field :object, type, null: true, resolve: ->(obj, args, ctx) { test_object } do + qt.field :item, type, null: true, resolve: ->(obj, args, ctx) { test_object } do authorize permissions end end @@ -78,7 +78,7 @@ describe 'Gitlab::Graphql::Authorization' do describe 'Field authorizations when field is a built in type' do let(:query_type) do query_factory do |query| - query.field :object, type, null: true, resolve: ->(obj, args, ctx) { test_object } + query.field :item, type, null: true, resolve: ->(obj, args, ctx) { test_object } end end @@ -131,7 +131,7 @@ describe 'Gitlab::Graphql::Authorization' do describe 'Type authorizations' do let(:query_type) do query_factory do |query| - query.field :object, type, null: true, resolve: ->(obj, args, ctx) { test_object } + query.field :item, type, null: true, resolve: ->(obj, args, ctx) { test_object } end end @@ -168,7 +168,7 @@ describe 'Gitlab::Graphql::Authorization' do let(:query_type) do query_factory do |query| - query.field :object, type, null: true, resolve: ->(obj, args, ctx) { test_object }, authorize: permission_2 + query.field :item, type, null: true, resolve: ->(obj, args, ctx) { test_object }, authorize: permission_2 end end @@ -176,7 +176,7 @@ describe 'Gitlab::Graphql::Authorization' do end describe 'type authorizations when applied to a relay connection' do - let(:query_string) { '{ object() { edges { node { name } } } }' } + let(:query_string) { '{ item() { edges { node { name } } } }' } let(:second_test_object) { double(name: 'Second thing') } let(:type) do @@ -187,11 +187,11 @@ describe 'Gitlab::Graphql::Authorization' do let(:query_type) do query_factory do |query| - query.field :object, type.connection_type, null: true, resolve: ->(obj, args, ctx) { [test_object, second_test_object] } + query.field :item, type.connection_type, null: true, resolve: ->(obj, args, ctx) { [test_object, second_test_object] } end end - subject { result.dig('object', 'edges') } + subject { result.dig('item', 'edges') } it 'returns only the elements visible to the user' do permit(permission_single) @@ -207,13 +207,13 @@ describe 'Gitlab::Graphql::Authorization' do describe 'limiting connections with multiple objects' do let(:query_type) do query_factory do |query| - query.field :object, type.connection_type, null: true, resolve: ->(obj, args, ctx) do + query.field :item, type.connection_type, null: true, resolve: ->(obj, args, ctx) do [test_object, second_test_object] end end end - let(:query_string) { '{ object(first: 1) { edges { node { name } } } }' } + let(:query_string) { '{ item(first: 1) { edges { node { name } } } }' } it 'only checks permissions for the first object' do expect(Ability).to receive(:allowed?).with(user, permission_single, test_object) { true } @@ -233,11 +233,11 @@ describe 'Gitlab::Graphql::Authorization' do let(:query_type) do query_factory do |query| - query.field :object, [type], null: true, resolve: ->(obj, args, ctx) { [test_object] } + query.field :item, [type], null: true, resolve: ->(obj, args, ctx) { [test_object] } end end - subject { result['object'].first } + subject { result['item'].first } include_examples 'authorization with a single permission' end diff --git a/spec/helpers/external_link_helper_spec.rb b/spec/helpers/external_link_helper_spec.rb new file mode 100644 index 00000000000..7fc4ef18731 --- /dev/null +++ b/spec/helpers/external_link_helper_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe ExternalLinkHelper do + include IconsHelper + + it 'returns external link with icon' do + expect(external_link('https://gitlab.com', 'https://gitlab.com').to_s) + .to eq('<a target="_blank" rel="noopener noreferrer" href="https://gitlab.com">https://gitlab.com <i aria-hidden="true" data-hidden="true" class="fa fa-external-link"></i></a>') + end + + it 'allows options when creating external link with icon' do + expect(external_link('https://gitlab.com', 'https://gitlab.com', { "data-foo": "bar", class: "externalLink" }).to_s) + .to eq('<a target="_blank" rel="noopener noreferrer" data-foo="bar" class="externalLink" href="https://gitlab.com">https://gitlab.com <i aria-hidden="true" data-hidden="true" class="fa fa-external-link"></i></a>') + end +end diff --git a/spec/helpers/icons_helper_spec.rb b/spec/helpers/icons_helper_spec.rb index f92b94a9583..950f951e22e 100644 --- a/spec/helpers/icons_helper_spec.rb +++ b/spec/helpers/icons_helper_spec.rb @@ -60,20 +60,19 @@ describe IconsHelper do non_existing = 'non_existing_icon_sprite' it 'raises in development mode' do - allow(Rails.env).to receive(:development?).and_return(true) + stub_rails_env('development') expect { sprite_icon(non_existing) }.to raise_error(ArgumentError, /is not a known icon/) end it 'raises in test mode' do - allow(Rails.env).to receive(:test?).and_return(true) + stub_rails_env('test') expect { sprite_icon(non_existing) }.to raise_error(ArgumentError, /is not a known icon/) end it 'does not raise in production mode' do - allow(Rails.env).to receive(:test?).and_return(false) - allow(Rails.env).to receive(:development?).and_return(false) + stub_rails_env('production') expect { sprite_icon(non_existing) }.not_to raise_error end diff --git a/spec/helpers/projects/error_tracking_helper_spec.rb b/spec/helpers/projects/error_tracking_helper_spec.rb index 7516a636c93..064b3ad21cb 100644 --- a/spec/helpers/projects/error_tracking_helper_spec.rb +++ b/spec/helpers/projects/error_tracking_helper_spec.rb @@ -6,21 +6,31 @@ describe Projects::ErrorTrackingHelper do include Gitlab::Routing.url_helpers set(:project) { create(:project) } + set(:current_user) { create(:user) } describe '#error_tracking_data' do + let(:can_enable_error_tracking) { true } let(:setting_path) { project_settings_operations_path(project) } let(:index_path) do project_error_tracking_index_path(project, format: :json) end + before do + allow(helper) + .to receive(:can?) + .with(current_user, :admin_operations, project) + .and_return(can_enable_error_tracking) + end + context 'without error_tracking_setting' do it 'returns frontend configuration' do - expect(error_tracking_data(project)).to eq( + expect(helper.error_tracking_data(current_user, project)).to match( 'index-path' => index_path, + 'user-can-enable-error-tracking' => 'true', 'enable-error-tracking-link' => setting_path, 'error-tracking-enabled' => 'false', - "illustration-path" => "/images/illustrations/cluster_popover.svg" + 'illustration-path' => match_asset_path('/assets/illustrations/cluster_popover.svg') ) end end @@ -36,7 +46,7 @@ describe Projects::ErrorTrackingHelper do end it 'show error tracking enabled' do - expect(error_tracking_data(project)).to include( + expect(helper.error_tracking_data(current_user, project)).to include( 'error-tracking-enabled' => 'true' ) end @@ -48,11 +58,21 @@ describe Projects::ErrorTrackingHelper do end it 'show error tracking not enabled' do - expect(error_tracking_data(project)).to include( + expect(helper.error_tracking_data(current_user, project)).to include( 'error-tracking-enabled' => 'false' ) end end end + + context 'when user is not maintainer' do + let(:can_enable_error_tracking) { false } + + it 'shows error tracking enablement as disabled' do + expect(helper.error_tracking_data(current_user, project)).to include( + 'user-can-enable-error-tracking' => 'false' + ) + end + end end end diff --git a/spec/helpers/version_check_helper_spec.rb b/spec/helpers/version_check_helper_spec.rb index e384e2bf9a0..edc0d64d031 100644 --- a/spec/helpers/version_check_helper_spec.rb +++ b/spec/helpers/version_check_helper_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe VersionCheckHelper do describe '#version_status_badge' do it 'returns nil if not dev environment and not enabled' do - allow(Rails.env).to receive(:production?) { false } + stub_rails_env('development') allow(Gitlab::CurrentSettings.current_application_settings).to receive(:version_check_enabled) { false } expect(helper.version_status_badge).to be(nil) @@ -11,7 +11,7 @@ describe VersionCheckHelper do context 'when production and enabled' do before do - allow(Rails.env).to receive(:production?) { true } + stub_rails_env('production') allow(Gitlab::CurrentSettings.current_application_settings).to receive(:version_check_enabled) { true } allow(VersionCheck).to receive(:url) { 'https://version.host.com/check.svg?gitlab_info=xxx' } end diff --git a/spec/javascripts/ide/components/preview/clientside_spec.js b/spec/javascripts/ide/components/preview/clientside_spec.js deleted file mode 100644 index b9bf5c51ffe..00000000000 --- a/spec/javascripts/ide/components/preview/clientside_spec.js +++ /dev/null @@ -1,363 +0,0 @@ -import Vue from 'vue'; -import { createComponentWithStore } from 'spec/helpers/vue_mount_component_helper'; -import { createStore } from '~/ide/stores'; -import Clientside from '~/ide/components/preview/clientside.vue'; -import timeoutPromise from 'spec/helpers/set_timeout_promise_helper'; -import { resetStore, file } from '../../helpers'; - -describe('IDE clientside preview', () => { - let vm; - let Component; - - beforeAll(() => { - Component = Vue.extend(Clientside); - }); - - beforeEach(done => { - const store = createStore(); - - Vue.set(store.state.entries, 'package.json', { - ...file('package.json'), - }); - Vue.set(store.state, 'currentProjectId', 'gitlab-ce'); - Vue.set(store.state.projects, 'gitlab-ce', { - visibility: 'public', - }); - - vm = createComponentWithStore(Component, store); - - spyOn(vm, 'getFileData').and.returnValue(Promise.resolve()); - spyOn(vm, 'getRawFileData').and.returnValue(Promise.resolve('')); - spyOn(vm, 'initManager'); - - vm.$mount(); - - timeoutPromise() - .then(() => vm.$nextTick()) - .then(done) - .catch(done.fail); - }); - - afterEach(() => { - vm.$destroy(); - resetStore(vm.$store); - }); - - describe('without main entry', () => { - it('creates sandpack manager', () => { - expect(vm.initManager).not.toHaveBeenCalled(); - }); - }); - - describe('with main entry', () => { - beforeEach(done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - - vm.$nextTick() - .then(() => vm.initPreview()) - .then(vm.$nextTick) - .then(done) - .catch(done.fail); - }); - - it('creates sandpack manager', () => { - expect(vm.initManager).toHaveBeenCalledWith( - '#ide-preview', - { - files: jasmine.any(Object), - entry: '/index.js', - showOpenInCodeSandbox: true, - }, - { - fileResolver: { - isFile: jasmine.any(Function), - readFile: jasmine.any(Function), - }, - }, - ); - }); - }); - - describe('computed', () => { - describe('normalizedEntries', () => { - beforeEach(done => { - vm.$store.state.entries['index.js'] = { - ...file('index.js'), - type: 'blob', - raw: 'test', - }; - vm.$store.state.entries['index2.js'] = { - ...file('index2.js'), - type: 'blob', - content: 'content', - }; - vm.$store.state.entries.tree = { - ...file('tree'), - type: 'tree', - }; - vm.$store.state.entries.empty = { - ...file('empty'), - type: 'blob', - }; - - vm.$nextTick(done); - }); - - it('returns flattened list of blobs with content', () => { - expect(vm.normalizedEntries).toEqual({ - '/index.js': { - code: 'test', - }, - '/index2.js': { - code: 'content', - }, - }); - }); - }); - - describe('mainEntry', () => { - it('returns false when package.json is empty', () => { - expect(vm.mainEntry).toBe(false); - }); - - it('returns main key from package.json', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - - vm.$nextTick(() => { - expect(vm.mainEntry).toBe('index.js'); - - done(); - }); - }); - }); - - describe('showPreview', () => { - it('returns false if no mainEntry', () => { - expect(vm.showPreview).toBe(false); - }); - - it('returns false if loading', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - vm.loading = true; - - vm.$nextTick(() => { - expect(vm.showPreview).toBe(false); - - done(); - }); - }); - - it('returns true if not loading and mainEntry exists', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - vm.loading = false; - - vm.$nextTick(() => { - expect(vm.showPreview).toBe(true); - - done(); - }); - }); - }); - - describe('showEmptyState', () => { - it('returns true if no mainEnry exists', () => { - expect(vm.showEmptyState).toBe(true); - }); - - it('returns false if loading', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - vm.loading = true; - - vm.$nextTick(() => { - expect(vm.showEmptyState).toBe(false); - - done(); - }); - }); - - it('returns false if not loading and mainEntry exists', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - vm.loading = false; - - vm.$nextTick(() => { - expect(vm.showEmptyState).toBe(false); - - done(); - }); - }); - }); - - describe('showOpenInCodeSandbox', () => { - it('returns true when visiblity is public', () => { - expect(vm.showOpenInCodeSandbox).toBe(true); - }); - - it('returns false when visiblity is private', done => { - vm.$store.state.projects['gitlab-ce'].visibility = 'private'; - - vm.$nextTick(() => { - expect(vm.showOpenInCodeSandbox).toBe(false); - - done(); - }); - }); - }); - - describe('sandboxOpts', () => { - beforeEach(done => { - vm.$store.state.entries['index.js'] = { - ...file('index.js'), - type: 'blob', - raw: 'test', - }; - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - - vm.$nextTick(done); - }); - - it('returns sandbox options', () => { - expect(vm.sandboxOpts).toEqual({ - files: { - '/index.js': { - code: 'test', - }, - '/package.json': { - code: '{"main":"index.js"}', - }, - }, - entry: '/index.js', - showOpenInCodeSandbox: true, - }); - }); - }); - }); - - describe('methods', () => { - describe('loadFileContent', () => { - it('calls getFileData', () => { - expect(vm.getFileData).toHaveBeenCalledWith({ - path: 'package.json', - makeFileActive: false, - }); - }); - - it('calls getRawFileData', () => { - expect(vm.getRawFileData).toHaveBeenCalledWith({ path: 'package.json' }); - }); - }); - - describe('update', () => { - beforeEach(() => { - jasmine.clock().install(); - - vm.sandpackReady = true; - vm.manager.updatePreview = jasmine.createSpy('updatePreview'); - vm.manager.listener = jasmine.createSpy('updatePreview'); - }); - - afterEach(() => { - jasmine.clock().uninstall(); - }); - - it('calls initPreview if manager is empty', () => { - spyOn(vm, 'initPreview'); - vm.manager = {}; - - vm.update(); - - jasmine.clock().tick(250); - - expect(vm.initPreview).toHaveBeenCalled(); - }); - - it('calls updatePreview', () => { - vm.update(); - - jasmine.clock().tick(250); - - expect(vm.manager.updatePreview).toHaveBeenCalledWith(vm.sandboxOpts); - }); - }); - }); - - describe('template', () => { - it('renders ide-preview element when showPreview is true', done => { - Vue.set( - vm.$store.state.entries['package.json'], - 'raw', - JSON.stringify({ - main: 'index.js', - }), - ); - vm.loading = false; - - vm.$nextTick(() => { - expect(vm.$el.querySelector('#ide-preview')).not.toBe(null); - done(); - }); - }); - - it('renders empty state', done => { - vm.loading = false; - - vm.$nextTick(() => { - expect(vm.$el.textContent).toContain( - 'Preview your web application using Web IDE client-side evaluation.', - ); - - done(); - }); - }); - - it('renders loading icon', done => { - vm.loading = true; - - vm.$nextTick(() => { - expect(vm.$el.querySelector('.loading-container')).not.toBe(null); - done(); - }); - }); - }); -}); diff --git a/spec/javascripts/jobs/components/job_app_spec.js b/spec/javascripts/jobs/components/job_app_spec.js index d3c1cf831bb..57ab1aa73f7 100644 --- a/spec/javascripts/jobs/components/job_app_spec.js +++ b/spec/javascripts/jobs/components/job_app_spec.js @@ -25,6 +25,7 @@ describe('Job App ', () => { terminalPath: 'jobs/123/terminal', pagePath: `${gl.TEST_HOST}jobs/123`, projectPath: 'user-name/project-name', + subscriptionsMoreMinutesUrl: 'https://customers.gitlab.com/buy_pipeline_minutes', logState: 'eyJvZmZzZXQiOjE3NDUxLCJuX29wZW5fdGFncyI6MCwiZmdfY29sb3IiOm51bGwsImJnX2NvbG9yIjpudWxsLCJzdHlsZV9tYXNrIjowfQ%3D%3D', }; diff --git a/spec/lib/api/support/git_access_actor_spec.rb b/spec/lib/api/support/git_access_actor_spec.rb new file mode 100644 index 00000000000..63f5966faea --- /dev/null +++ b/spec/lib/api/support/git_access_actor_spec.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::Support::GitAccessActor do + let(:user) { nil } + let(:key) { nil } + + subject { described_class.new(user: user, key: key) } + + describe '.from_params' do + context 'with params that are valid' do + it 'returns an instance of API::Support::GitAccessActor' do + params = { key_id: create(:key).id } + + expect(described_class.from_params(params)).to be_instance_of(described_class) + end + end + + context 'with params that are invalid' do + it 'returns nil' do + expect(described_class.from_params({})).to be_nil + end + end + end + + describe 'attributes' do + describe '#user' do + context 'when initialized with a User' do + let(:user) { create(:user) } + + it 'returns the User' do + expect(subject.user).to eq(user) + end + end + + context 'when initialized with a Key' do + let(:user_for_key) { create(:user) } + let(:key) { create(:key, user: user_for_key) } + + it 'returns the User associated to the Key' do + expect(subject.user).to eq(user_for_key) + end + end + end + end + + describe '#key_or_user' do + context 'when params contains a :key_id' do + it 'is an instance of Key' do + key = create(:key) + params = { key_id: key.id } + + expect(described_class.from_params(params).key_or_user).to eq(key) + end + end + + context 'when params contains a :user_id' do + it 'is an instance of User' do + user = create(:user) + params = { user_id: user.id } + + expect(described_class.from_params(params).key_or_user).to eq(user) + end + end + + context 'when params contains a :username' do + it 'is an instance of User (via UserFinder)' do + user = create(:user) + params = { username: user.username } + + expect(described_class.from_params(params).key_or_user).to eq(user) + end + end + end + + describe '#username' do + context 'when initialized with a User' do + let(:user) { create(:user) } + + it 'returns the username' do + expect(subject.username).to eq(user.username) + end + end + + context 'when initialized with a Key' do + let(:key) { create(:key, user: user_for_key) } + + context 'that has no User associated' do + let(:user_for_key) { nil } + + it 'returns nil' do + expect(subject.username).to be_nil + end + end + + context 'that has a User associated' do + let(:user_for_key) { create(:user) } + + it 'returns the username of the User associated to the Key' do + expect(subject.username).to eq(user_for_key.username) + end + end + end + end + + describe '#update_last_used_at!' do + context 'when initialized with a User' do + let(:user) { create(:user) } + + it 'does nothing' do + expect(user).not_to receive(:update_last_used_at) + + subject.update_last_used_at! + end + end + + context 'when initialized with a Key' do + let(:key) { create(:key) } + + it 'updates update_last_used_at' do + expect(key).to receive(:update_last_used_at) + + subject.update_last_used_at! + end + end + end +end diff --git a/spec/lib/banzai/filter/asset_proxy_filter_spec.rb b/spec/lib/banzai/filter/asset_proxy_filter_spec.rb index b7f45421b2a..0c4ccbf28f4 100644 --- a/spec/lib/banzai/filter/asset_proxy_filter_spec.rb +++ b/spec/lib/banzai/filter/asset_proxy_filter_spec.rb @@ -36,6 +36,17 @@ describe Banzai::Filter::AssetProxyFilter do expect(Gitlab.config.asset_proxy.whitelist).to eq %w(gitlab.com *.mydomain.com) expect(Gitlab.config.asset_proxy.domain_regexp).to eq /^(gitlab\.com|.*?\.mydomain\.com)$/i end + + context 'when whitelist is empty' do + it 'defaults to the install domain' do + stub_application_setting(asset_proxy_enabled: true) + stub_application_setting(asset_proxy_whitelist: []) + + described_class.initialize_settings + + expect(Gitlab.config.asset_proxy.whitelist).to eq [Gitlab.config.gitlab.host] + end + end end context 'when properly configured' do diff --git a/spec/lib/gitlab/checks/diff_check_spec.rb b/spec/lib/gitlab/checks/diff_check_spec.rb index a341dfa5636..b9134b8d6ab 100644 --- a/spec/lib/gitlab/checks/diff_check_spec.rb +++ b/spec/lib/gitlab/checks/diff_check_spec.rb @@ -20,9 +20,8 @@ describe Gitlab::Checks::DiffCheck do allow(project).to receive(:lfs_enabled?).and_return(false) end - it 'skips the validation' do - expect(subject).not_to receive(:validate_diff) - expect(subject).not_to receive(:validate_file_paths) + it 'does not invoke :lfs_file_locks_validation' do + expect(subject).not_to receive(:lfs_file_locks_validation) subject.validate! end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 91c559dcd9b..cf496b79a62 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -50,6 +50,32 @@ module Gitlab end end + describe 'interruptible entry' do + describe 'interruptible job' do + let(:config) do + YAML.dump(rspec: { script: 'rspec', interruptible: true }) + end + + it { expect(subject[:interruptible]).to be_truthy } + end + + describe 'interruptible job with default value' do + let(:config) do + YAML.dump(rspec: { script: 'rspec' }) + end + + it { expect(subject).not_to have_key(:interruptible) } + end + + describe 'uninterruptible job' do + let(:config) do + YAML.dump(rspec: { script: 'rspec', interruptible: false }) + end + + it { expect(subject[:interruptible]).to be_falsy } + end + end + describe 'retry entry' do context 'when retry count is specified' do let(:config) do diff --git a/spec/lib/gitlab/danger/helper_spec.rb b/spec/lib/gitlab/danger/helper_spec.rb index 2990594c538..710564b7540 100644 --- a/spec/lib/gitlab/danger/helper_spec.rb +++ b/spec/lib/gitlab/danger/helper_spec.rb @@ -168,9 +168,13 @@ describe Gitlab::Danger::Helper do 'Gemfile.lock' | :backend 'Procfile' | :backend 'Rakefile' | :backend - '.gitlab-ci.yml' | :backend 'FOO_VERSION' | :backend + '.gitlab-ci.yml' | :engineering_productivity + '.gitlab/ci/cng.gitlab-ci.yml' | :engineering_productivity + '.gitlab/ci/ee-specific-checks.gitlab-ci.yml' | :engineering_productivity + 'lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml' | :backend + 'ee/FOO_VERSION' | :unknown 'db/schema.rb' | :database diff --git a/spec/lib/gitlab/danger/teammate_spec.rb b/spec/lib/gitlab/danger/teammate_spec.rb index afbc3896a70..ca036390bde 100644 --- a/spec/lib/gitlab/danger/teammate_spec.rb +++ b/spec/lib/gitlab/danger/teammate_spec.rb @@ -65,6 +65,18 @@ describe Gitlab::Danger::Teammate do expect(subject.reviewer?(project, :test, labels)).to be_falsey end end + + context 'when role is Backend Engineer, Engineering Productivity' do + let(:role) { 'Backend Engineer, Engineering Productivity' } + + it '#reviewer? returns true' do + expect(subject.reviewer?(project, :engineering_productivity, labels)).to be_truthy + end + + it '#maintainer? returns false' do + expect(subject.maintainer?(project, :engineering_productivity, labels)).to be_falsey + end + end end end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index cff4eb398bf..49f92f14559 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -1283,33 +1283,19 @@ describe Gitlab::Database::MigrationHelpers do describe '#perform_background_migration_inline?' do it 'returns true in a test environment' do - allow(Rails.env) - .to receive(:test?) - .and_return(true) + stub_rails_env('test') expect(model.perform_background_migration_inline?).to eq(true) end it 'returns true in a development environment' do - allow(Rails.env) - .to receive(:test?) - .and_return(false) - - allow(Rails.env) - .to receive(:development?) - .and_return(true) + stub_rails_env('development') expect(model.perform_background_migration_inline?).to eq(true) end it 'returns false in a production environment' do - allow(Rails.env) - .to receive(:test?) - .and_return(false) - - allow(Rails.env) - .to receive(:development?) - .and_return(false) + stub_rails_env('production') expect(model.perform_background_migration_inline?).to eq(false) end diff --git a/spec/lib/gitlab/favicon_spec.rb b/spec/lib/gitlab/favicon_spec.rb index 63c26e29d73..d221f39c2ed 100644 --- a/spec/lib/gitlab/favicon_spec.rb +++ b/spec/lib/gitlab/favicon_spec.rb @@ -1,8 +1,6 @@ require 'spec_helper' RSpec.describe Gitlab::Favicon, :request_store do - include RailsHelpers - describe '.main' do it 'defaults to favicon.png' do stub_rails_env('production') diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index 99d563e03ec..1c5f72a4396 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -265,7 +265,7 @@ describe Gitlab::GitalyClient do context 'in production and when RequestStore is enabled', :request_store do before do - allow(Rails.env).to receive(:production?).and_return(true) + stub_rails_env('production') end context 'when the maximum number of calls is enforced by a feature flag' do diff --git a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb index 7a7ae373058..aada9285b31 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb @@ -32,7 +32,8 @@ describe Gitlab::Graphql::Authorize::AuthorizeFieldService do let(:presented_type) { double('parent type', object: presented_object) } let(:query_type) { GraphQL::ObjectType.new } let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} - let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: { current_user: current_user }, object: nil) } + let(:query_context) { OpenStruct.new(schema: schema) } + let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema, context: query_context), values: { current_user: current_user }, object: nil) } subject(:resolved) { service.authorized_resolve.call(presented_type, {}, context) } context 'scalar types' do diff --git a/spec/lib/gitlab/graphql/connections/keyset_connection_spec.rb b/spec/lib/gitlab/graphql/connections/keyset_connection_spec.rb index fefa2881b18..4eb121794e1 100644 --- a/spec/lib/gitlab/graphql/connections/keyset_connection_spec.rb +++ b/spec/lib/gitlab/graphql/connections/keyset_connection_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::Graphql::Connections::KeysetConnection do end def encoded_property(value) - Base64.strict_encode64(value.to_s) + Base64Bp.urlsafe_encode64(value.to_s, padding: false) end describe '#cursor_from_nodes' do diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 516e62c4728..d34c6d2421b 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -329,6 +329,7 @@ CommitStatus: - failure_reason - scheduled_at - upstream_pipeline_id +- interruptible Ci::Variable: - id - project_id @@ -467,6 +468,7 @@ ProtectedBranch: - name - created_at - updated_at +- code_owner_approval_required ProtectedTag: - id - project_id diff --git a/spec/lib/gitlab/metrics/transaction_spec.rb b/spec/lib/gitlab/metrics/transaction_spec.rb index e70fde09edd..45e74597a2e 100644 --- a/spec/lib/gitlab/metrics/transaction_spec.rb +++ b/spec/lib/gitlab/metrics/transaction_spec.rb @@ -204,17 +204,17 @@ describe Gitlab::Metrics::Transaction do end it 'allows tracking of custom tags' do - transaction.add_event(:meow, animal: 'cat') + transaction.add_event(:bau, animal: 'dog') - expect(metric.tags).to eq(event: :meow, animal: 'cat') + expect(metric.tags).to eq(event: :bau, animal: 'dog') end context 'with sensitive tags' do before do - transaction.add_event(:meow, **sensitive_tags.merge(sane: 'yes')) + transaction.add_event(:baubau, **sensitive_tags.merge(sane: 'yes')) end - it_behaves_like 'tag filter', event: :meow, sane: 'yes' + it_behaves_like 'tag filter', event: :baubau, sane: 'yes' end end diff --git a/spec/lib/gitlab/metrics/web_transaction_spec.rb b/spec/lib/gitlab/metrics/web_transaction_spec.rb index 2b35f07cc0d..21a762dbf25 100644 --- a/spec/lib/gitlab/metrics/web_transaction_spec.rb +++ b/spec/lib/gitlab/metrics/web_transaction_spec.rb @@ -253,11 +253,11 @@ describe Gitlab::Metrics::WebTransaction do end it 'allows tracking of custom tags' do - transaction.add_event(:meow, animal: 'cat') + transaction.add_event(:bau, animal: 'dog') metric = transaction.metrics[0] - expect(metric.tags).to eq(event: :meow, animal: 'cat') + expect(metric.tags).to eq(event: :bau, animal: 'dog') end end end diff --git a/spec/lib/gitlab/query_limiting/transaction_spec.rb b/spec/lib/gitlab/query_limiting/transaction_spec.rb index 39d5a575efc..4e906314b5a 100644 --- a/spec/lib/gitlab/query_limiting/transaction_spec.rb +++ b/spec/lib/gitlab/query_limiting/transaction_spec.rb @@ -86,9 +86,7 @@ describe Gitlab::QueryLimiting::Transaction do it 'returns false in a production environment' do transaction = described_class.new - expect(Rails.env) - .to receive(:test?) - .and_return(false) + stub_rails_env('production') expect(transaction.raise_error?).to eq(false) end diff --git a/spec/lib/gitlab/query_limiting_spec.rb b/spec/lib/gitlab/query_limiting_spec.rb index f0d0340cd6e..e9c6bbc35a3 100644 --- a/spec/lib/gitlab/query_limiting_spec.rb +++ b/spec/lib/gitlab/query_limiting_spec.rb @@ -9,14 +9,14 @@ describe Gitlab::QueryLimiting do end it 'returns true in a development environment' do - allow(Rails.env).to receive(:development?).and_return(true) + stub_rails_env('development') + stub_rails_env('development') expect(described_class.enable?).to eq(true) end it 'returns false on GitLab.com' do - expect(Rails.env).to receive(:development?).and_return(false) - expect(Rails.env).to receive(:test?).and_return(false) + stub_rails_env('production') allow(Gitlab).to receive(:com?).and_return(true) expect(described_class.enable?).to eq(false) @@ -24,8 +24,7 @@ describe Gitlab::QueryLimiting do it 'returns false in a non GitLab.com' do allow(Gitlab).to receive(:com?).and_return(false) - expect(Rails.env).to receive(:development?).and_return(false) - expect(Rails.env).to receive(:test?).and_return(false) + stub_rails_env('production') expect(described_class.enable?).to eq(false) end diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index df8a1f82f81..6d1d7e48326 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -4,80 +4,114 @@ require 'spec_helper' describe Gitlab::UrlBlocker do + include StubRequests + describe '#validate!' do + subject { described_class.validate!(import_url) } + + shared_examples 'validates URI and hostname' do + it 'runs the url validations' do + uri, hostname = subject + + expect(uri).to eq(Addressable::URI.parse(expected_uri)) + expect(hostname).to eq(expected_hostname) + end + end + context 'when URI is nil' do let(:import_url) { nil } - it 'returns no URI and hostname' do - uri, hostname = described_class.validate!(import_url) - - expect(uri).to be(nil) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { nil } + let(:expected_hostname) { nil } end end context 'when URI is internal' do let(:import_url) { 'http://localhost' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url) - - expect(uri).to eq(Addressable::URI.parse('http://[::1]')) - expect(hostname).to eq('localhost') + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://[::1]' } + let(:expected_hostname) { 'localhost' } end end context 'when the URL hostname is a domain' do - let(:import_url) { 'https://example.org' } + context 'when domain can be resolved' do + let(:import_url) { 'https://example.org' } - it 'returns URI and hostname' do - uri, hostname = described_class.validate!(import_url) + before do + stub_dns(import_url, ip_address: '93.184.216.34') + end - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to eq('example.org') + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'https://93.184.216.34' } + let(:expected_hostname) { 'example.org' } + end + end + + context 'when domain cannot be resolved' do + let(:import_url) { 'http://foobar.x' } + + it 'raises an error' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect { subject }.to raise_error(described_class::BlockedUrlError) + end end end context 'when the URL hostname is an IP address' do let(:import_url) { 'https://93.184.216.34' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end + + context 'when the address is invalid' do + let(:import_url) { 'http://1.1.1.1.1' } - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to be(nil) + it 'raises an error' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect { subject }.to raise_error(described_class::BlockedUrlError) + end end end context 'disabled DNS rebinding protection' do + subject { described_class.validate!(import_url, dns_rebind_protection: false) } + context 'when URI is internal' do let(:import_url) { 'http://localhost' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) - - expect(uri).to eq(Addressable::URI.parse('http://localhost')) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end context 'when the URL hostname is a domain' do let(:import_url) { 'https://example.org' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end - expect(uri).to eq(Addressable::URI.parse('https://example.org')) - expect(hostname).to eq(nil) + context 'when domain can be resolved' do + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end end - context 'when it cannot be resolved' do + context 'when domain cannot be resolved' do let(:import_url) { 'http://foobar.x' } - it 'raises error' do - stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - - expect { described_class.validate!(import_url) }.to raise_error(described_class::BlockedUrlError) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end end @@ -85,20 +119,17 @@ describe Gitlab::UrlBlocker do context 'when the URL hostname is an IP address' do let(:import_url) { 'https://93.184.216.34' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) - - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end context 'when it is invalid' do let(:import_url) { 'http://1.1.1.1.1' } - it 'raises an error' do - stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - - expect { described_class.validate!(import_url) }.to raise_error(described_class::BlockedUrlError) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end end diff --git a/spec/lib/gitlab_spec.rb b/spec/lib/gitlab_spec.rb index 74d4b12a070..589dac61528 100644 --- a/spec/lib/gitlab_spec.rb +++ b/spec/lib/gitlab_spec.rb @@ -3,8 +3,6 @@ require 'spec_helper' describe Gitlab do - include RailsHelpers - describe '.root' do it 'returns the root path of the app' do expect(described_class.root).to eq(Pathname.new(File.expand_path('../..', __dir__))) diff --git a/spec/models/clusters/applications/jupyter_spec.rb b/spec/models/clusters/applications/jupyter_spec.rb index 1b39328752d..e1eee014567 100644 --- a/spec/models/clusters/applications/jupyter_spec.rb +++ b/spec/models/clusters/applications/jupyter_spec.rb @@ -81,27 +81,45 @@ describe Clusters::Applications::Jupyter do end describe '#files' do - let(:application) { create(:clusters_applications_jupyter) } + let(:cluster) { create(:cluster, :with_installed_helm, :provided_by_gcp, :project) } + let(:application) { create(:clusters_applications_jupyter, cluster: cluster) } let(:values) { subject[:'values.yaml'] } subject { application.files } - it 'includes valid values' do - expect(values).to include('ingress') - expect(values).to include('hub') - expect(values).to include('rbac') - expect(values).to include('proxy') - expect(values).to include('auth') - expect(values).to include('singleuser') - expect(values).to match(/clientId: '?#{application.oauth_application.uid}/) - expect(values).to match(/callbackUrl: '?#{application.callback_url}/) - expect(values).to include("gitlabProjectIdWhitelist:\n - #{application.cluster.project.id}") - expect(values).to include("c.GitLabOAuthenticator.scope = ['api read_repository write_repository']") - expect(values).to match(/GITLAB_HOST: '?#{Gitlab.config.gitlab.host}/) + context 'when cluster belongs to a project' do + it 'includes valid values' do + expect(values).to include('ingress') + expect(values).to include('hub') + expect(values).to include('rbac') + expect(values).to include('proxy') + expect(values).to include('auth') + expect(values).to include('singleuser') + expect(values).to match(/clientId: '?#{application.oauth_application.uid}/) + expect(values).to match(/callbackUrl: '?#{application.callback_url}/) + expect(values).to include("gitlabProjectIdWhitelist:\n - #{application.cluster.project.id}") + expect(values).to include("c.GitLabOAuthenticator.scope = ['api read_repository write_repository']") + expect(values).to match(/GITLAB_HOST: '?#{Gitlab.config.gitlab.host}/) + expect(values).to match(/GITLAB_CLUSTER_ID: '?#{application.cluster.id}/) + end end - context 'when cluster belongs to a project' do - it 'sets GitLab project id' do + context 'when cluster belongs to a group' do + let(:group) { create(:group) } + let(:cluster) { create(:cluster, :with_installed_helm, :provided_by_gcp, :group, groups: [group]) } + + it 'includes valid values' do + expect(values).to include('ingress') + expect(values).to include('hub') + expect(values).to include('rbac') + expect(values).to include('proxy') + expect(values).to include('auth') + expect(values).to include('singleuser') + expect(values).to match(/clientId: '?#{application.oauth_application.uid}/) + expect(values).to match(/callbackUrl: '?#{application.callback_url}/) + expect(values).to include("gitlabGroupWhitelist:\n - #{group.to_param}") + expect(values).to include("c.GitLabOAuthenticator.scope = ['api read_repository write_repository']") + expect(values).to match(/GITLAB_HOST: '?#{Gitlab.config.gitlab.host}/) expect(values).to match(/GITLAB_CLUSTER_ID: '?#{application.cluster.id}/) end end diff --git a/spec/models/concerns/cacheable_attributes_spec.rb b/spec/models/concerns/cacheable_attributes_spec.rb index da46effe411..d8f940a808e 100644 --- a/spec/models/concerns/cacheable_attributes_spec.rb +++ b/spec/models/concerns/cacheable_attributes_spec.rb @@ -131,7 +131,7 @@ describe CacheableAttributes do context 'in production environment' do before do - expect(Rails.env).to receive(:production?).and_return(true) + stub_rails_env('production') end it 'returns an uncached record and logs a warning' do @@ -143,7 +143,7 @@ describe CacheableAttributes do context 'in other environments' do before do - expect(Rails.env).to receive(:production?).and_return(false) + stub_rails_env('development') end it 'returns an uncached record and logs a warning' do diff --git a/spec/models/concerns/has_status_spec.rb b/spec/models/concerns/has_status_spec.rb index a217dc42537..09fb2fff521 100644 --- a/spec/models/concerns/has_status_spec.rb +++ b/spec/models/concerns/has_status_spec.rb @@ -261,6 +261,18 @@ describe HasStatus do end end + describe '.alive_or_scheduled' do + subject { CommitStatus.alive_or_scheduled } + + %i[running pending preparing created scheduled].each do |status| + it_behaves_like 'containing the job', status + end + + %i[failed success canceled skipped].each do |status| + it_behaves_like 'not containing the job', status + end + end + describe '.created_or_pending' do subject { CommitStatus.created_or_pending } diff --git a/spec/models/concerns/sha_attribute_spec.rb b/spec/models/concerns/sha_attribute_spec.rb index a4a81ae126d..0d4dbfb215e 100644 --- a/spec/models/concerns/sha_attribute_spec.rb +++ b/spec/models/concerns/sha_attribute_spec.rb @@ -17,7 +17,7 @@ describe ShaAttribute do describe '#sha_attribute' do context 'when in non-production' do before do - allow(Rails.env).to receive(:production?).and_return(false) + stub_rails_env('development') end context 'when the table exists' do @@ -76,7 +76,7 @@ describe ShaAttribute do context 'when in production' do before do - allow(Rails.env).to receive(:production?).and_return(true) + stub_rails_env('production') end it 'defines a SHA attribute' do diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb index 28630f7d3fe..c73ade3f896 100644 --- a/spec/models/internal_id_spec.rb +++ b/spec/models/internal_id_spec.rb @@ -106,7 +106,8 @@ describe InternalId do end it 'always attempts to generate internal IDs in production mode' do - allow(Rails.env).to receive(:test?).and_return(false) + stub_rails_env('production') + val = rand(1..100) generator = double(generate: val) expect(InternalId::InternalIdGenerator).to receive(:new).and_return(generator) diff --git a/spec/requests/api/graphql/gitlab_schema_spec.rb b/spec/requests/api/graphql/gitlab_schema_spec.rb index 28676bb02f4..e1eb7c7f738 100644 --- a/spec/requests/api/graphql/gitlab_schema_spec.rb +++ b/spec/requests/api/graphql/gitlab_schema_spec.rb @@ -120,7 +120,7 @@ describe 'GitlabSchema configurations' do query_string: query, variables: {}.to_s, complexity: 181, - depth: 0, + depth: 13, duration: 7 } diff --git a/spec/requests/api/graphql/mutations/merge_requests/set_wip_spec.rb b/spec/requests/api/graphql/mutations/merge_requests/set_wip_spec.rb index 3a8a2bae939..bbc477ba485 100644 --- a/spec/requests/api/graphql/mutations/merge_requests/set_wip_spec.rb +++ b/spec/requests/api/graphql/mutations/merge_requests/set_wip_spec.rb @@ -13,7 +13,16 @@ describe 'Setting WIP status of a merge request' do project_path: project.full_path, iid: merge_request.iid.to_s } - graphql_mutation(:merge_request_set_wip, variables.merge(input), "clientMutationId\nerrors\nmergeRequest { id\ntitle }") + graphql_mutation(:merge_request_set_wip, variables.merge(input), + <<-QL.strip_heredoc + clientMutationId + errors + mergeRequest { + id + title + } + QL + ) end def mutation_response diff --git a/spec/requests/rack_attack_global_spec.rb b/spec/requests/rack_attack_global_spec.rb index 478f09a7881..cf459ba99c1 100644 --- a/spec/requests/rack_attack_global_spec.rb +++ b/spec/requests/rack_attack_global_spec.rb @@ -20,11 +20,6 @@ describe 'Rack Attack global throttles' do let(:period_in_seconds) { 10000 } let(:period) { period_in_seconds.seconds } - let(:url_that_does_not_require_authentication) { '/users/sign_in' } - let(:url_that_requires_authentication) { '/dashboard/snippets' } - let(:url_api_internal) { '/api/v4/internal/check' } - let(:api_partial_url) { '/todos' } - around do |example| # Instead of test environment's :null_store so the throttles can increment Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new @@ -35,112 +30,10 @@ describe 'Rack Attack global throttles' do Rack::Attack.cache.store = Rails.cache end - # Requires let variables: - # * throttle_setting_prefix (e.g. "throttle_authenticated_api" or "throttle_authenticated_web") - # * get_args - # * other_user_get_args - shared_examples_for 'rate-limited token-authenticated requests' do - before do - # Set low limits - settings_to_set[:"#{throttle_setting_prefix}_requests_per_period"] = requests_per_period - settings_to_set[:"#{throttle_setting_prefix}_period_in_seconds"] = period_in_seconds - end - - context 'when the throttle is enabled' do - before do - settings_to_set[:"#{throttle_setting_prefix}_enabled"] = true - stub_application_setting(settings_to_set) - end - - it 'rejects requests over the rate limit' do - # At first, allow requests under the rate limit. - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - # the last straw - expect_rejection { get(*get_args) } - end - - it 'allows requests after throttling and then waiting for the next period' do - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - expect_rejection { get(*get_args) } - - Timecop.travel(period.from_now) do - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - expect_rejection { get(*get_args) } - end - end - - it 'counts requests from different users separately, even from the same IP' do - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - # would be over the limit if this wasn't a different user - get(*other_user_get_args) - expect(response).to have_http_status 200 - end - - it 'counts all requests from the same user, even via different IPs' do - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - expect_any_instance_of(Rack::Attack::Request).to receive(:ip).and_return('1.2.3.4') - - expect_rejection { get(*get_args) } - end - - it 'logs RackAttack info into structured logs' do - requests_per_period.times do - get(*get_args) - expect(response).to have_http_status 200 - end - - arguments = { - message: 'Rack_Attack', - env: :throttle, - remote_ip: '127.0.0.1', - request_method: 'GET', - path: get_args.first, - user_id: user.id, - username: user.username - } - - expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once - - expect_rejection { get(*get_args) } - end - end - - context 'when the throttle is disabled' do - before do - settings_to_set[:"#{throttle_setting_prefix}_enabled"] = false - stub_application_setting(settings_to_set) - end - - it 'allows requests over the rate limit' do - (1 + requests_per_period).times do - get(*get_args) - expect(response).to have_http_status 200 - end - end - end - end - describe 'unauthenticated requests' do + let(:url_that_does_not_require_authentication) { '/users/sign_in' } + let(:url_api_internal) { '/api/v4/internal/check' } + before do # Set low limits settings_to_set[:throttle_unauthenticated_requests_per_period] = requests_per_period @@ -245,6 +138,7 @@ describe 'Rack Attack global throttles' do let(:other_user) { create(:user) } let(:other_user_token) { create(:personal_access_token, user: other_user) } let(:throttle_setting_prefix) { 'throttle_authenticated_api' } + let(:api_partial_url) { '/todos' } context 'with the token in the query string' do let(:get_args) { [api(api_partial_url, personal_access_token: token)] } @@ -265,10 +159,13 @@ describe 'Rack Attack global throttles' do let(:user) { create(:user) } let(:application) { Doorkeeper::Application.create!(name: "MyApp", redirect_uri: "https://app.com", owner: user) } let(:token) { Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: "api") } + let(:other_user) { create(:user) } let(:other_user_application) { Doorkeeper::Application.create!(name: "MyApp", redirect_uri: "https://app.com", owner: other_user) } let(:other_user_token) { Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: other_user.id, scopes: "api") } + let(:throttle_setting_prefix) { 'throttle_authenticated_api' } + let(:api_partial_url) { '/todos' } context 'with the token in the query string' do let(:get_args) { [api(api_partial_url, oauth_access_token: token)] } @@ -299,110 +196,11 @@ describe 'Rack Attack global throttles' do end describe 'web requests authenticated with regular login' do + let(:throttle_setting_prefix) { 'throttle_authenticated_web' } let(:user) { create(:user) } + let(:url_that_requires_authentication) { '/dashboard/snippets' } - before do - login_as(user) - - # Set low limits - settings_to_set[:throttle_authenticated_web_requests_per_period] = requests_per_period - settings_to_set[:throttle_authenticated_web_period_in_seconds] = period_in_seconds - end - - context 'when the throttle is enabled' do - before do - settings_to_set[:throttle_authenticated_web_enabled] = true - stub_application_setting(settings_to_set) - end - - it 'rejects requests over the rate limit' do - # At first, allow requests under the rate limit. - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - # the last straw - expect_rejection { get url_that_requires_authentication } - end - - it 'allows requests after throttling and then waiting for the next period' do - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - expect_rejection { get url_that_requires_authentication } - - Timecop.travel(period.from_now) do - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - expect_rejection { get url_that_requires_authentication } - end - end - - it 'counts requests from different users separately, even from the same IP' do - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - # would be over the limit if this wasn't a different user - login_as(create(:user)) - - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - it 'counts all requests from the same user, even via different IPs' do - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - expect_any_instance_of(Rack::Attack::Request).to receive(:ip).and_return('1.2.3.4') - - expect_rejection { get url_that_requires_authentication } - end - - it 'logs RackAttack info into structured logs' do - requests_per_period.times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - - arguments = { - message: 'Rack_Attack', - env: :throttle, - remote_ip: '127.0.0.1', - request_method: 'GET', - path: '/dashboard/snippets', - user_id: user.id, - username: user.username - } - - expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once - - get url_that_requires_authentication - end - end - - context 'when the throttle is disabled' do - before do - settings_to_set[:throttle_authenticated_web_enabled] = false - stub_application_setting(settings_to_set) - end - - it 'allows requests over the rate limit' do - (1 + requests_per_period).times do - get url_that_requires_authentication - expect(response).to have_http_status 200 - end - end - end + it_behaves_like 'rate-limited web authenticated requests' end def api_get_args_with_token_headers(partial_url, token_headers) diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index fad865a4811..6cec93a53fd 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -220,11 +220,11 @@ describe Ci::CreatePipelineService do expect(pipeline_on_previous_commit.reload).to have_attributes(status: 'canceled', auto_canceled_by_id: pipeline.id) end - it 'does not cancel running outdated pipelines' do + it 'cancels running outdated pipelines' do pipeline_on_previous_commit.run - execute_service + head_pipeline = execute_service - expect(pipeline_on_previous_commit.reload).to have_attributes(status: 'running', auto_canceled_by_id: nil) + expect(pipeline_on_previous_commit.reload).to have_attributes(status: 'canceled', auto_canceled_by_id: head_pipeline.id) end it 'cancel created outdated pipelines' do @@ -243,6 +243,202 @@ describe Ci::CreatePipelineService do expect(pending_pipeline.reload).to have_attributes(status: 'pending', auto_canceled_by_id: nil) end + + context 'when the interruptible attribute is' do + context 'not defined' do + before do + config = YAML.dump(rspec: { script: 'echo' }) + stub_ci_pipeline_yaml_file(config) + end + + it 'is cancelable' do + pipeline = execute_service + + expect(pipeline.builds.find_by(name: 'rspec').interruptible).to be_nil + end + end + + context 'set to true' do + before do + config = YAML.dump(rspec: { script: 'echo', interruptible: true }) + stub_ci_pipeline_yaml_file(config) + end + + it 'is cancelable' do + pipeline = execute_service + + expect(pipeline.builds.find_by(name: 'rspec').interruptible).to be_truthy + end + end + + context 'set to false' do + before do + config = YAML.dump(rspec: { script: 'echo', interruptible: false }) + stub_ci_pipeline_yaml_file(config) + end + + it 'is not cancelable' do + pipeline = execute_service + + expect(pipeline.builds.find_by(name: 'rspec').interruptible).to be_falsy + end + end + + context 'not defined, but an environment is' do + before do + config = YAML.dump(rspec: { script: 'echo', environment: { name: "review/$CI_COMMIT_REF_NAME" } }) + stub_ci_pipeline_yaml_file(config) + end + + it 'is not cancelable' do + pipeline = execute_service + + expect(pipeline.builds.find_by(name: 'rspec').interruptible).to be_nil + end + end + + context 'overriding the environment definition' do + before do + config = YAML.dump(rspec: { script: 'echo', environment: { name: "review/$CI_COMMIT_REF_NAME" }, interruptible: true }) + stub_ci_pipeline_yaml_file(config) + end + + it 'is cancelable' do + pipeline = execute_service + + expect(pipeline.builds.find_by(name: 'rspec').interruptible).to be_truthy + end + end + end + + context 'interruptible builds' do + before do + stub_ci_pipeline_yaml_file(YAML.dump(config)) + end + + let(:config) do + { + stages: %w[stage1 stage2 stage3 stage4], + + build_1_1: { + stage: 'stage1', + script: 'echo' + }, + build_1_2: { + stage: 'stage1', + script: 'echo', + interruptible: true + }, + build_2_1: { + stage: 'stage2', + script: 'echo', + when: 'delayed', + start_in: '10 minutes' + }, + build_3_1: { + stage: 'stage3', + script: 'echo', + interruptible: false + }, + build_4_1: { + stage: 'stage4', + script: 'echo' + } + } + end + + it 'properly configures interruptible status' do + interruptible_status = + pipeline_on_previous_commit + .builds + .joins(:metadata) + .pluck(:name, 'ci_builds_metadata.interruptible') + + expect(interruptible_status).to contain_exactly( + ['build_1_1', nil], + ['build_1_2', true], + ['build_2_1', nil], + ['build_3_1', false], + ['build_4_1', nil] + ) + end + + context 'when only interruptible builds are running' do + context 'when build marked explicitly by interruptible is running' do + it 'cancels running outdated pipelines' do + pipeline_on_previous_commit + .builds + .find_by_name('build_1_2') + .run! + + pipeline + + expect(pipeline_on_previous_commit.reload).to have_attributes( + status: 'canceled', auto_canceled_by_id: pipeline.id) + end + end + + context 'when build that is not marked as interruptible is running' do + it 'cancels running outdated pipelines' do + pipeline_on_previous_commit + .builds + .find_by_name('build_2_1') + .tap(&:enqueue!) + .run! + + pipeline + + expect(pipeline_on_previous_commit.reload).to have_attributes( + status: 'canceled', auto_canceled_by_id: pipeline.id) + end + end + end + + context 'when an uninterruptible build is running' do + it 'does not cancel running outdated pipelines' do + pipeline_on_previous_commit + .builds + .find_by_name('build_3_1') + .tap(&:enqueue!) + .run! + + pipeline + + expect(pipeline_on_previous_commit.reload).to have_attributes( + status: 'running', auto_canceled_by_id: nil) + end + end + + context 'when an build is waiting on an interruptible scheduled task' do + it 'cancels running outdated pipelines' do + allow(Ci::BuildScheduleWorker).to receive(:perform_at) + + pipeline_on_previous_commit + .builds + .find_by_name('build_2_1') + .schedule! + + pipeline + + expect(pipeline_on_previous_commit.reload).to have_attributes( + status: 'canceled', auto_canceled_by_id: pipeline.id) + end + end + + context 'when a uninterruptible build has finished' do + it 'does not cancel running outdated pipelines' do + pipeline_on_previous_commit + .builds + .find_by_name('build_3_1') + .success! + + pipeline + + expect(pipeline_on_previous_commit.reload).to have_attributes( + status: 'running', auto_canceled_by_id: nil) + end + end + end end context 'auto-cancel disabled' do diff --git a/spec/services/clusters/applications/check_installation_progress_service_spec.rb b/spec/services/clusters/applications/check_installation_progress_service_spec.rb index 464a67649ff..02fd4b63c89 100644 --- a/spec/services/clusters/applications/check_installation_progress_service_spec.rb +++ b/spec/services/clusters/applications/check_installation_progress_service_spec.rb @@ -142,7 +142,11 @@ describe Clusters::Applications::CheckInstallationProgressService, '#execute' do end it 'removes the installation POD' do - expect(service).to receive(:remove_installation_pod).once + expect_any_instance_of(Gitlab::Kubernetes::Helm::Api) + .to receive(:delete_pod!) + .with(kind_of(String)) + .once + expect(service).to receive(:remove_installation_pod).and_call_original service.execute end diff --git a/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb b/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb index 1a9f7089c3d..68ad0208226 100644 --- a/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb +++ b/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb @@ -47,11 +47,15 @@ describe Clusters::Applications::CheckUninstallProgressService do context 'when installation POD succeeded' do let(:phase) { Gitlab::Kubernetes::Pod::SUCCEEDED } before do + expect_any_instance_of(Gitlab::Kubernetes::Helm::Api) + .to receive(:delete_pod!) + .with(kind_of(String)) + .once expect(service).to receive(:pod_phase).once.and_return(phase) end it 'removes the installation POD' do - expect(service).to receive(:remove_installation_pod).once + expect(service).to receive(:remove_uninstallation_pod).and_call_original service.execute end @@ -76,7 +80,7 @@ describe Clusters::Applications::CheckUninstallProgressService do end it 'still removes the installation POD' do - expect(service).to receive(:remove_installation_pod).once + expect(service).to receive(:remove_uninstallation_pod).and_call_original service.execute end diff --git a/spec/services/clusters/applications/create_service_spec.rb b/spec/services/clusters/applications/create_service_spec.rb index bb86a742f0e..8dd573c3698 100644 --- a/spec/services/clusters/applications/create_service_spec.rb +++ b/spec/services/clusters/applications/create_service_spec.rb @@ -147,12 +147,12 @@ describe Clusters::Applications::CreateService do using RSpec::Parameterized::TableSyntax - where(:application, :association, :allowed, :pre_create_helm) do - 'helm' | :application_helm | true | false - 'ingress' | :application_ingress | true | true - 'runner' | :application_runner | true | true - 'prometheus' | :application_prometheus | true | true - 'jupyter' | :application_jupyter | false | true + where(:application, :association, :allowed, :pre_create_helm, :pre_create_ingress) do + 'helm' | :application_helm | true | false | false + 'ingress' | :application_ingress | true | true | false + 'runner' | :application_runner | true | true | false + 'prometheus' | :application_prometheus | true | true | false + 'jupyter' | :application_jupyter | true | true | true end with_them do @@ -160,6 +160,7 @@ describe Clusters::Applications::CreateService do klass = "Clusters::Applications::#{application.titleize}" allow_any_instance_of(klass.constantize).to receive(:make_scheduled!).and_call_original create(:clusters_applications_helm, :installed, cluster: cluster) if pre_create_helm + create(:clusters_applications_ingress, :installed, cluster: cluster, external_hostname: 'example.com') if pre_create_ingress end let(:params) { { application: application } } diff --git a/spec/services/merge_requests/push_options_handler_service_spec.rb b/spec/services/merge_requests/push_options_handler_service_spec.rb index a27fea0c90f..ff4cdd3e7e2 100644 --- a/spec/services/merge_requests/push_options_handler_service_spec.rb +++ b/spec/services/merge_requests/push_options_handler_service_spec.rb @@ -13,6 +13,9 @@ describe MergeRequests::PushOptionsHandlerService do let(:target_branch) { 'feature' } let(:title) { 'my title' } let(:description) { 'my description' } + let(:label1) { 'mylabel1' } + let(:label2) { 'mylabel2' } + let(:label3) { 'mylabel3' } let(:new_branch_changes) { "#{Gitlab::Git::BLANK_SHA} 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/#{source_branch}" } let(:existing_branch_changes) { "d14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/#{source_branch}" } let(:deleted_branch_changes) { "d14d6c0abdd253381df51a723d58691b2ee1ab08 #{Gitlab::Git::BLANK_SHA} refs/heads/#{source_branch}" } @@ -122,6 +125,16 @@ describe MergeRequests::PushOptionsHandlerService do end end + shared_examples_for 'a service that can change labels of a merge request' do |count| + subject(:last_mr) { MergeRequest.last } + + it 'changes label count' do + service.execute + + expect(last_mr.label_ids.count).to eq(count) + end + end + shared_examples_for 'a service that does not create a merge request' do it do expect { service.execute }.not_to change { MergeRequest.count } @@ -504,6 +517,138 @@ describe MergeRequests::PushOptionsHandlerService do end end + describe '`label` push option' do + let(:push_options) { { label: { label1 => 1, label2 => 1 } } } + + context 'with a new branch' do + let(:changes) { new_branch_changes } + + it_behaves_like 'a service that does not create a merge request' + + it 'adds an error to the service' do + error = "A merge_request.create push option is required to create a merge request for branch #{source_branch}" + + service.execute + + expect(service.errors).to include(error) + end + + context 'when coupled with the `create` push option' do + let(:push_options) { { create: true, label: { label1 => 1, label2 => 1 } } } + + it_behaves_like 'a service that can create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 2 + end + end + + context 'with an existing branch but no open MR' do + let(:changes) { existing_branch_changes } + + it_behaves_like 'a service that does not create a merge request' + + it 'adds an error to the service' do + error = "A merge_request.create push option is required to create a merge request for branch #{source_branch}" + + service.execute + + expect(service.errors).to include(error) + end + + context 'when coupled with the `create` push option' do + let(:push_options) { { create: true, label: { label1 => 1, label2 => 1 } } } + + it_behaves_like 'a service that can create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 2 + end + end + + context 'with an existing branch that has a merge request open' do + let(:changes) { existing_branch_changes } + let!(:merge_request) { create(:merge_request, source_project: project, source_branch: source_branch)} + + it_behaves_like 'a service that does not create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 2 + end + + context 'with a deleted branch' do + let(:changes) { deleted_branch_changes } + + it_behaves_like 'a service that does nothing' + end + + context 'with the project default branch' do + let(:changes) { default_branch_changes } + + it_behaves_like 'a service that does nothing' + end + end + + describe '`unlabel` push option' do + let(:push_options) { { label: { label1 => 1, label2 => 1 }, unlabel: { label1 => 1, label3 => 1 } } } + + context 'with a new branch' do + let(:changes) { new_branch_changes } + + it_behaves_like 'a service that does not create a merge request' + + it 'adds an error to the service' do + error = "A merge_request.create push option is required to create a merge request for branch #{source_branch}" + + service.execute + + expect(service.errors).to include(error) + end + + context 'when coupled with the `create` push option' do + let(:push_options) { { create: true, label: { label1 => 1, label2 => 1 }, unlabel: { label1 => 1, label3 => 1 } } } + + it_behaves_like 'a service that can create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 1 + end + end + + context 'with an existing branch but no open MR' do + let(:changes) { existing_branch_changes } + + it_behaves_like 'a service that does not create a merge request' + + it 'adds an error to the service' do + error = "A merge_request.create push option is required to create a merge request for branch #{source_branch}" + + service.execute + + expect(service.errors).to include(error) + end + + context 'when coupled with the `create` push option' do + let(:push_options) { { create: true, label: { label1 => 1, label2 => 1 }, unlabel: { label1 => 1, label3 => 1 } } } + + it_behaves_like 'a service that can create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 1 + end + end + + context 'with an existing branch that has a merge request open' do + let(:changes) { existing_branch_changes } + let!(:merge_request) { create(:merge_request, source_project: project, source_branch: source_branch)} + + it_behaves_like 'a service that does not create a merge request' + it_behaves_like 'a service that can change labels of a merge request', 1 + end + + context 'with a deleted branch' do + let(:changes) { deleted_branch_changes } + + it_behaves_like 'a service that does nothing' + end + + context 'with the project default branch' do + let(:changes) { default_branch_changes } + + it_behaves_like 'a service that does nothing' + end + end + describe 'multiple pushed branches' do let(:push_options) { { create: true } } let(:changes) do diff --git a/spec/services/quick_actions/interpret_service_spec.rb b/spec/services/quick_actions/interpret_service_spec.rb index c9714964fc9..6ca0a3fa448 100644 --- a/spec/services/quick_actions/interpret_service_spec.rb +++ b/spec/services/quick_actions/interpret_service_spec.rb @@ -970,34 +970,6 @@ describe QuickActions::InterpretService do let(:issuable) { merge_request } end - it_behaves_like 'due command' do - let(:content) { '/due 2016-08-28' } - let(:issuable) { issue } - end - - it_behaves_like 'due command' do - let(:content) { '/due tomorrow' } - let(:issuable) { issue } - let(:expected_date) { Date.tomorrow } - end - - it_behaves_like 'due command' do - let(:content) { '/due 5 days from now' } - let(:issuable) { issue } - let(:expected_date) { 5.days.from_now.to_date } - end - - it_behaves_like 'due command' do - let(:content) { '/due in 2 days' } - let(:issuable) { issue } - let(:expected_date) { 2.days.from_now.to_date } - end - - it_behaves_like 'empty command' do - let(:content) { '/due foo bar' } - let(:issuable) { issue } - end - it_behaves_like 'empty command' do let(:content) { '/due 2016-08-28' } let(:issuable) { merge_request } @@ -1131,6 +1103,39 @@ describe QuickActions::InterpretService do end end + context '/due command' do + it 'returns invalid date format message when the due date is invalid' do + issue = build(:issue, project: project) + + _, _, message = service.execute('/due invalid date', issue) + + expect(message).to eq('Failed to set due date because the date format is invalid.') + end + + it_behaves_like 'due command' do + let(:content) { '/due 2016-08-28' } + let(:issuable) { issue } + end + + it_behaves_like 'due command' do + let(:content) { '/due tomorrow' } + let(:issuable) { issue } + let(:expected_date) { Date.tomorrow } + end + + it_behaves_like 'due command' do + let(:content) { '/due 5 days from now' } + let(:issuable) { issue } + let(:expected_date) { 5.days.from_now.to_date } + end + + it_behaves_like 'due command' do + let(:content) { '/due in 2 days' } + let(:issuable) { issue } + let(:expected_date) { 2.days.from_now.to_date } + end + end + context '/copy_metadata command' do let(:todo_label) { create(:label, project: project, title: 'To Do') } let(:inreview_label) { create(:label, project: project, title: 'In Review') } diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6ce76af556f..47f09bf14d0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -109,6 +109,7 @@ RSpec.configure do |config| config.include PolicyHelpers, type: :policy config.include MemoryUsageHelper config.include ExpectRequestWithStatus, type: :request + config.include RailsHelpers if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. diff --git a/spec/support/shared_examples/requests/rack_attack_shared_examples.rb b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb new file mode 100644 index 00000000000..afc6f59b773 --- /dev/null +++ b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true +# +# Requires let variables: +# * throttle_setting_prefix: "throttle_authenticated_api", "throttle_authenticated_web", "throttle_protected_paths" +# * get_args +# * other_user_get_args +# * requests_per_period +# * period_in_seconds +# * period +shared_examples_for 'rate-limited token-authenticated requests' do + before do + # Set low limits + settings_to_set[:"#{throttle_setting_prefix}_requests_per_period"] = requests_per_period + settings_to_set[:"#{throttle_setting_prefix}_period_in_seconds"] = period_in_seconds + end + + context 'when the throttle is enabled' do + before do + settings_to_set[:"#{throttle_setting_prefix}_enabled"] = true + stub_application_setting(settings_to_set) + end + + it 'rejects requests over the rate limit' do + # At first, allow requests under the rate limit. + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + # the last straw + expect_rejection { get(*get_args) } + end + + it 'allows requests after throttling and then waiting for the next period' do + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + expect_rejection { get(*get_args) } + + Timecop.travel(period.from_now) do + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + expect_rejection { get(*get_args) } + end + end + + it 'counts requests from different users separately, even from the same IP' do + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + # would be over the limit if this wasn't a different user + get(*other_user_get_args) + expect(response).to have_http_status 200 + end + + it 'counts all requests from the same user, even via different IPs' do + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + expect_any_instance_of(Rack::Attack::Request).to receive(:ip).and_return('1.2.3.4') + + expect_rejection { get(*get_args) } + end + + it 'logs RackAttack info into structured logs' do + requests_per_period.times do + get(*get_args) + expect(response).to have_http_status 200 + end + + arguments = { + message: 'Rack_Attack', + env: :throttle, + remote_ip: '127.0.0.1', + request_method: 'GET', + path: get_args.first, + user_id: user.id, + username: user.username + } + + expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once + + expect_rejection { get(*get_args) } + end + end + + context 'when the throttle is disabled' do + before do + settings_to_set[:"#{throttle_setting_prefix}_enabled"] = false + stub_application_setting(settings_to_set) + end + + it 'allows requests over the rate limit' do + (1 + requests_per_period).times do + get(*get_args) + expect(response).to have_http_status 200 + end + end + end +end + +# Requires let variables: +# * throttle_setting_prefix: "throttle_authenticated_web" or "throttle_protected_paths" +# * user +# * url_that_requires_authentication +# * requests_per_period +# * period_in_seconds +# * period +shared_examples_for 'rate-limited web authenticated requests' do + before do + login_as(user) + + # Set low limits + settings_to_set[:"#{throttle_setting_prefix}_requests_per_period"] = requests_per_period + settings_to_set[:"#{throttle_setting_prefix}_period_in_seconds"] = period_in_seconds + end + + context 'when the throttle is enabled' do + before do + settings_to_set[:"#{throttle_setting_prefix}_enabled"] = true + stub_application_setting(settings_to_set) + end + + it 'rejects requests over the rate limit' do + # At first, allow requests under the rate limit. + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + # the last straw + expect_rejection { get url_that_requires_authentication } + end + + it 'allows requests after throttling and then waiting for the next period' do + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + expect_rejection { get url_that_requires_authentication } + + Timecop.travel(period.from_now) do + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + expect_rejection { get url_that_requires_authentication } + end + end + + it 'counts requests from different users separately, even from the same IP' do + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + # would be over the limit if this wasn't a different user + login_as(create(:user)) + + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + it 'counts all requests from the same user, even via different IPs' do + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + expect_any_instance_of(Rack::Attack::Request).to receive(:ip).and_return('1.2.3.4') + + expect_rejection { get url_that_requires_authentication } + end + + it 'logs RackAttack info into structured logs' do + requests_per_period.times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + + arguments = { + message: 'Rack_Attack', + env: :throttle, + remote_ip: '127.0.0.1', + request_method: 'GET', + path: '/dashboard/snippets', + user_id: user.id, + username: user.username + } + + expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once + + get url_that_requires_authentication + end + end + + context 'when the throttle is disabled' do + before do + settings_to_set[:"#{throttle_setting_prefix}_enabled"] = false + stub_application_setting(settings_to_set) + end + + it 'allows requests over the rate limit' do + (1 + requests_per_period).times do + get url_that_requires_authentication + expect(response).to have_http_status 200 + end + end + end +end diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index e6e4d9504d9..2f3fc7839c1 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -57,7 +57,7 @@ describe 'gitlab:gitaly namespace rake task' do stub_env('CI', false) FileUtils.mkdir_p(clone_path) expect(Dir).to receive(:chdir).with(clone_path).and_call_original - allow(Rails.env).to receive(:test?).and_return(false) + stub_rails_env('development') end context 'gmake is available' do @@ -93,7 +93,7 @@ describe 'gitlab:gitaly namespace rake task' do end before do - allow(Rails.env).to receive(:test?).and_return(true) + stub_rails_env('test') end it 'calls make in the gitaly directory with --no-deployment flag for bundle' do diff --git a/spec/validators/addressable_url_validator_spec.rb b/spec/validators/addressable_url_validator_spec.rb index 387e84b2d04..6927a1f67a1 100644 --- a/spec/validators/addressable_url_validator_spec.rb +++ b/spec/validators/addressable_url_validator_spec.rb @@ -92,6 +92,15 @@ describe AddressableUrlValidator do expect(badge.errors).to be_empty expect(badge.link_url).to eq('https://127.0.0.1') end + + it 'allows urls that cannot be resolved' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + badge.link_url = 'http://foobar.x' + + subject + + expect(badge.errors).to be_empty + end end context 'when message is set' do @@ -312,4 +321,32 @@ describe AddressableUrlValidator do end end end + + context 'when dns_rebind_protection is' do + let(:not_resolvable_url) { 'http://foobar.x' } + let(:validator) { described_class.new(attributes: [:link_url], dns_rebind_protection: dns_value) } + + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + badge.link_url = not_resolvable_url + + subject + end + + context 'true' do + let(:dns_value) { true } + + it 'raises error' do + expect(badge.errors).to be_present + end + end + + context 'false' do + let(:dns_value) { false } + + it 'allows urls that cannot be resolved' do + expect(badge.errors).to be_empty + end + end + end end diff --git a/tmp/.gitignore b/tmp/.gitignore new file mode 100644 index 00000000000..a383b23fb78 --- /dev/null +++ b/tmp/.gitignore @@ -0,0 +1,5 @@ +* +!*/ +!.gitignore +!.gitkeep +/tests/ |
