diff options
67 files changed, 963 insertions, 719 deletions
diff --git a/.eslintignore b/.eslintignore index 9a5e15c86ae..f364771e54f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -9,6 +9,6 @@ /scripts/ /tmp/ /vendor/ -jest.config.js +jest.config.*.js karma.config.js webpack.config.js diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 0028358987a..da74f4de6c2 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -238,6 +238,18 @@ jest: junit: junit_jest.xml parallel: 2 +jest-integration: + extends: .frontend-job-base + script: + - date + - yarn jest:integration --ci + needs: ["frontend-fixtures"] + cache: + key: jest-integration + paths: + - tmp/cache/jest/ + policy: pull-push + jest-as-if-foss: extends: - .jest-base @@ -307,10 +319,10 @@ webpack-dev-server: key: files: - yarn.lock - - config/webpack.vendor.config.js - prefix: "v2" + prefix: "v1" paths: - node_modules/ + - tmp/cache/webpack-dlls/ script: - source scripts/utils.sh - retry yarn install --frozen-lockfile diff --git a/GITLAB_ELASTICSEARCH_INDEXER_VERSION b/GITLAB_ELASTICSEARCH_INDEXER_VERSION index ccbccc3dc62..276cbf9e285 100644 --- a/GITLAB_ELASTICSEARCH_INDEXER_VERSION +++ b/GITLAB_ELASTICSEARCH_INDEXER_VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 @@ -324,7 +324,6 @@ gem 'derailed_benchmarks', require: false # Metrics group :metrics do gem 'method_source', '~> 0.8', require: false - gem 'influxdb', '~> 0.2', require: false # Prometheus gem 'prometheus-client-mmap', '~> 0.10.0' diff --git a/Gemfile.lock b/Gemfile.lock index 1ed450b62c6..9a9c7c62dbc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -153,7 +153,6 @@ GEM activemodel (>= 4.0.0) activesupport (>= 4.0.0) mime-types (>= 1.16) - cause (0.1) character_set (1.1.2) charlock_holmes (0.7.6) childprocess (3.0.0) @@ -535,9 +534,6 @@ GEM i18n_data (0.8.0) icalendar (2.4.1) ice_nine (0.11.2) - influxdb (0.2.3) - cause - json invisible_captcha (0.12.1) rails (>= 3.2.0) ipaddress (0.8.3) @@ -1276,7 +1272,6 @@ DEPENDENCIES html2text httparty (~> 0.16.4) icalendar - influxdb (~> 0.2) invisible_captcha (~> 0.12.1) jira-ruby (~> 2.0.0) js_regex (~> 3.1) diff --git a/app/assets/javascripts/confidential_merge_request/components/dropdown.vue b/app/assets/javascripts/confidential_merge_request/components/dropdown.vue index 444640980af..92a5423d5ea 100644 --- a/app/assets/javascripts/confidential_merge_request/components/dropdown.vue +++ b/app/assets/javascripts/confidential_merge_request/components/dropdown.vue @@ -39,7 +39,7 @@ export default { <template> <gl-dropdown toggle-class="d-flex align-items-center w-100" class="w-100"> - <template slot="button-content"> + <template #button-content> <span class="str-truncated-100 mr-2"> <icon name="lock" /> {{ dropdownText }} diff --git a/app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss b/app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss index 86dffb4d7df..2a164d961d2 100644 --- a/app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss +++ b/app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss @@ -1,3 +1,6 @@ +// ------- +// Please see `app/assets/stylesheets/page_bundles/ide_themes/README.md` for a guide on contributing new themes +// ------- .ide.theme-dark { a:not(.btn) { color: var(--ide-link-color); diff --git a/app/assets/stylesheets/page_bundles/ide_themes/README.md b/app/assets/stylesheets/page_bundles/ide_themes/README.md new file mode 100644 index 00000000000..535179cc4c2 --- /dev/null +++ b/app/assets/stylesheets/page_bundles/ide_themes/README.md @@ -0,0 +1,53 @@ +# Web IDE Themes + +Web IDE currently supports 5 syntax highlighting themes based on themes from the user's profile preferences: + +* White +* Dark +* Monokai +* Solarized Dark +* Solarized Light + +Currently, the Web IDE supports the white theme by default, and the dark theme by the introduction of CSS +variables. + +The Web IDE automatically adds an appropriate theme class to the `ide.vue` component based on the current syntax +highlighting theme. Below are those theme classes, which come from the `gon.user_color_scheme` global setting: + +| # | Color Scheme | `gon.user_color_scheme` | Theme class | +|---|-----------------|-------------------------|-------------------------| +| 1 | White | `"white"` | `.theme-white` | +| 2 | Dark | `"dark"` | `.theme-dark` | +| 3 | Monokai | `"monokai"` | `.theme-monokai` | +| 4 | Solarized Dark | `"solarized-dark"` | `.theme-solarized-dark` | +| 5 | Solarized Light | `"solarized-light"` | `.theme-solarized-light` | +| 6 | None | `"none"` | `.theme-none` | + +## Adding New Themes (SCSS) + +To add a new theme, follow the following steps: + +1. Pick a theme from the table above, lets say **Solarized Dark**. +2. Create a new file in this folder called `_solarized_dark.scss`. +3. Copy over all the CSS variables from `_dark.scss` to `_solarized_dark.scss` and assign them your own values. + Put them under the selector `.ide.theme-solarized-dark`. +4. Import this newly created SCSS file in `ide.scss` file in the parent directory. +5. To make sure the variables apply to to your theme, add the selector `.ide.theme-solarized-dark` to the top + of `_ide_theme_overrides.scss` file. The file should now look like this: + + ```scss + .ide.theme-dark, + .ide.theme-solarized-dark { + /* file contents */ + } + ``` + + This step is temporary until all CSS variables in that file have their + default values assigned. +6. That's it! Raise a merge request with your newly added theme. + +## Modifying Monaco Themes + +Monaco themes are defined in Javascript and are stored in the `app/assets/javascripts/ide/lib/themes/` directory. +To modify any syntax highlighting colors or to synchronize the theme colors with syntax highlighting colors, you +can modify the files in that directory directly. diff --git a/app/assets/stylesheets/page_bundles/ide_themes/_dark.scss b/app/assets/stylesheets/page_bundles/ide_themes/_dark.scss index 809abc42a69..43e91eb283d 100644 --- a/app/assets/stylesheets/page_bundles/ide_themes/_dark.scss +++ b/app/assets/stylesheets/page_bundles/ide_themes/_dark.scss @@ -1,3 +1,6 @@ +// ------- +// Please see `app/assets/stylesheets/page_bundles/ide_themes/README.md` for a guide on contributing new themes +// ------- .ide.theme-dark { --ide-border-color: #1d1f21; --ide-highlight-accent: #fff; diff --git a/app/controllers/concerns/metrics_dashboard.rb b/app/controllers/concerns/metrics_dashboard.rb index c7fb3c6d403..1aea0e294a5 100644 --- a/app/controllers/concerns/metrics_dashboard.rb +++ b/app/controllers/concerns/metrics_dashboard.rb @@ -18,7 +18,7 @@ module MetricsDashboard if result result[:all_dashboards] = all_dashboards if include_all_dashboards? - result[:metrics_data] = metrics_data(project_for_dashboard, environment_for_dashboard) if project_for_dashboard && environment_for_dashboard + result[:metrics_data] = metrics_data(project_for_dashboard, environment_for_dashboard) end respond_to do |format| diff --git a/app/helpers/environments_helper.rb b/app/helpers/environments_helper.rb index 21b2111baf7..e7b561af3da 100644 --- a/app/helpers/environments_helper.rb +++ b/app/helpers/environments_helper.rb @@ -22,33 +22,13 @@ module EnvironmentsHelper end def metrics_data(project, environment) - { - "settings-path" => edit_project_service_path(project, 'prometheus'), - "clusters-path" => project_clusters_path(project), - "current-environment-name" => environment.name, - "documentation-path" => help_page_path('administration/monitoring/prometheus/index.md'), - "empty-getting-started-svg-path" => image_path('illustrations/monitoring/getting_started.svg'), - "empty-loading-svg-path" => image_path('illustrations/monitoring/loading.svg'), - "empty-no-data-svg-path" => image_path('illustrations/monitoring/no_data.svg'), - "empty-no-data-small-svg-path" => image_path('illustrations/chart-empty-state-small.svg'), - "empty-unable-to-connect-svg-path" => image_path('illustrations/monitoring/unable_to_connect.svg'), - "metrics-endpoint" => additional_metrics_project_environment_path(project, environment, format: :json), - "dashboards-endpoint" => project_performance_monitoring_dashboards_path(project, format: :json), - "dashboard-endpoint" => metrics_dashboard_project_environment_path(project, environment, format: :json), - "deployments-endpoint" => project_environment_deployments_path(project, environment, format: :json), - "default-branch" => project.default_branch, - "project-path" => project_path(project), - "tags-path" => project_tags_path(project), - "has-metrics" => "#{environment.has_metrics?}", - "prometheus-status" => "#{environment.prometheus_status}", - "external-dashboard-url" => project.metrics_setting_external_dashboard_url, - "environment-state" => "#{environment.state}", - "custom-metrics-path" => project_prometheus_metrics_path(project), - "validate-query-path" => validate_query_project_prometheus_metrics_path(project), - "custom-metrics-available" => "#{custom_metrics_available?(project)}", - "alerts-endpoint" => project_prometheus_alerts_path(project, environment_id: environment.id, format: :json), - "prometheus-alerts-available" => "#{can?(current_user, :read_prometheus_alerts, project)}" - } + metrics_data = {} + metrics_data.merge!(project_metrics_data(project)) if project + metrics_data.merge!(environment_metrics_data(environment)) if environment + metrics_data.merge!(project_and_environment_metrics_data(project, environment)) if project && environment + metrics_data.merge!(static_metrics_data) + + metrics_data end def environment_logs_data(project, environment) @@ -63,6 +43,60 @@ module EnvironmentsHelper def can_destroy_environment?(environment) can?(current_user, :destroy_environment, environment) end + + private + + def project_metrics_data(project) + return {} unless project + + { + 'settings-path' => edit_project_service_path(project, 'prometheus'), + 'clusters-path' => project_clusters_path(project), + 'dashboards-endpoint' => project_performance_monitoring_dashboards_path(project, format: :json), + 'default-branch' => project.default_branch, + 'project-path' => project_path(project), + 'tags-path' => project_tags_path(project), + 'external-dashboard-url' => project.metrics_setting_external_dashboard_url, + 'custom-metrics-path' => project_prometheus_metrics_path(project), + 'validate-query-path' => validate_query_project_prometheus_metrics_path(project), + 'custom-metrics-available' => "#{custom_metrics_available?(project)}", + 'prometheus-alerts-available' => "#{can?(current_user, :read_prometheus_alerts, project)}" + } + end + + def environment_metrics_data(environment) + return {} unless environment + + { + 'current-environment-name' => environment.name, + 'has-metrics' => "#{environment.has_metrics?}", + 'prometheus-status' => "#{environment.prometheus_status}", + 'environment-state' => "#{environment.state}" + } + end + + def project_and_environment_metrics_data(project, environment) + return {} unless project && environment + + { + 'metrics-endpoint' => additional_metrics_project_environment_path(project, environment, format: :json), + 'dashboard-endpoint' => metrics_dashboard_project_environment_path(project, environment, format: :json), + 'deployments-endpoint' => project_environment_deployments_path(project, environment, format: :json), + 'alerts-endpoint' => project_prometheus_alerts_path(project, environment_id: environment.id, format: :json) + + } + end + + def static_metrics_data + { + 'documentation-path' => help_page_path('administration/monitoring/prometheus/index.md'), + 'empty-getting-started-svg-path' => image_path('illustrations/monitoring/getting_started.svg'), + 'empty-loading-svg-path' => image_path('illustrations/monitoring/loading.svg'), + 'empty-no-data-svg-path' => image_path('illustrations/monitoring/no_data.svg'), + 'empty-no-data-small-svg-path' => image_path('illustrations/chart-empty-state-small.svg'), + 'empty-unable-to-connect-svg-path' => image_path('illustrations/monitoring/unable_to_connect.svg') + } + end end EnvironmentsHelper.prepend_if_ee('::EE::EnvironmentsHelper') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index d13f0a1a000..498d6823cc7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -266,6 +266,8 @@ class ApplicationSetting < ApplicationRecord validates :email_restrictions, untrusted_regexp: true + validates :hashed_storage_enabled, inclusion: { in: [true], message: _("Hashed storage can't be disabled anymore for new projects") } + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 76882dfcb0d..1e92a47ab49 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -166,6 +166,10 @@ module Ci end end + def dependency_variables + [] + end + private def cross_project_params diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 04f6387b519..7f64ea7dd97 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -33,7 +33,6 @@ module Ci scheduler_failure: 2 }.freeze - CODE_NAVIGATION_JOB_NAME = 'code_navigation' DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' has_one :deployment, as: :deployable, class_name: 'Deployment' @@ -605,6 +604,14 @@ module Ci Ci::FreezePeriodStatus.new(project: project).execute end + def dependency_variables + return [] if all_dependencies.empty? + + Gitlab::Ci::Variables::Collection.new.concat( + Ci::JobVariable.where(job: all_dependencies).dotenv_source + ) + end + def features { trace_sections: true } end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index cc13f279c4d..e4e0f55d5f4 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -161,7 +161,6 @@ module CacheMarkdownField define_method(invalidation_method) do changed_fields = changed_attributes.keys invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] - invalidations.delete(markdown_field.to_s) if changed_fields.include?("#{markdown_field}_html") !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 5ff537a7837..b19f4518fb2 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -18,6 +18,7 @@ module Ci variables.concat(deployment_variables(environment: environment)) variables.concat(yaml_variables) variables.concat(user_variables) + variables.concat(dependency_variables) if Feature.enabled?(:ci_dependency_variables, project) variables.concat(secret_group_variables) variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 7c17456f7fc..50443048274 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -280,6 +280,9 @@ class ProjectPolicy < BasePolicy enable :read_prometheus enable :read_environment enable :read_deployment + end + + rule { ~anonymous & can?(:metrics_dashboard) }.policy do enable :create_metrics_user_starred_dashboard enable :read_metrics_user_starred_dashboard end @@ -426,11 +429,27 @@ class ProjectPolicy < BasePolicy rule { builds_disabled | repository_disabled }.policy do prevent(*create_read_update_admin_destroy(:build)) prevent(*create_read_update_admin_destroy(:pipeline_schedule)) - prevent(*create_read_update_admin_destroy(:environment)) prevent(*create_read_update_admin_destroy(:cluster)) prevent(*create_read_update_admin_destroy(:deployment)) end + # Enabling `read_environment` specifically for the condition of `metrics_dashboard_allowed` is + # necessary due to the route for metrics dashboard requiring an environment id. + # This will be addressed in https://gitlab.com/gitlab-org/gitlab/-/issues/213833 when + # environments and metrics are decoupled and these rules will be removed. + + rule { (builds_disabled | repository_disabled) & ~metrics_dashboard_allowed}.policy do + prevent(*create_read_update_admin_destroy(:environment)) + end + + rule { (builds_disabled | repository_disabled) & metrics_dashboard_allowed}.policy do + prevent :create_environment + prevent :update_environment + prevent :admin_environment + prevent :destroy_environment + enable :read_environment + end + # There's two separate cases when builds_disabled is true: # 1. When internal CI is disabled - builds_disabled && internal_builds_disabled # - We do not prevent the user from accessing Pipelines to allow them to access external CI diff --git a/app/views/admin/application_settings/_repository_storage.html.haml b/app/views/admin/application_settings/_repository_storage.html.haml index c3ae39ddd48..6fabafe3fc1 100644 --- a/app/views/admin/application_settings/_repository_storage.html.haml +++ b/app/views/admin/application_settings/_repository_storage.html.haml @@ -6,10 +6,10 @@ %h4= _("Hashed repository storage paths") .form-group .form-check - = f.check_box :hashed_storage_enabled, class: 'form-check-input qa-hashed-storage-checkbox' + = f.check_box :hashed_storage_enabled, class: 'form-check-input qa-hashed-storage-checkbox', disabled: @application_setting.hashed_storage_enabled? = f.label :hashed_storage_enabled, _("Use hashed storage"), class: 'label-bold form-check-label' .form-text.text-muted - = _("Use hashed storage paths for newly created and renamed repositories. Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Repository URL changes and may improve disk I/O performance.") + = _("Use hashed storage paths for newly created and renamed repositories. Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Repository URL changes and may improve disk I/O performance. (Always enabled since 13.0)") .sub-section %h4= _("Storage nodes for new repositories") .form-group diff --git a/app/views/profiles/show.html.haml b/app/views/profiles/show.html.haml index afb307a8018..43fc9150e99 100644 --- a/app/views/profiles/show.html.haml +++ b/app/views/profiles/show.html.haml @@ -104,7 +104,7 @@ - else = f.text_field :location, label: s_('Profiles|Location'), class: 'input-lg', placeholder: s_("Profiles|City, country") = f.text_field :job_title, class: 'input-md' - = render 'shared/profiles/organization', { form: f, user: @user } + = f.text_field :organization, label: s_('Profiles|Organization'), class: 'input-md', help: s_("Profiles|Who you represent or work for") = f.text_area :bio, label: s_('Profiles|Bio'), rows: 4, maxlength: 250, help: s_("Profiles|Tell us about yourself in fewer than 250 characters") %hr %h5= s_("Private profile") diff --git a/app/views/shared/milestones/_top.html.haml b/app/views/shared/milestones/_top.html.haml index 8d911d4247e..5f53e6316af 100644 --- a/app/views/shared/milestones/_top.html.haml +++ b/app/views/shared/milestones/_top.html.haml @@ -21,10 +21,10 @@ %table.table %thead %tr - %th Project - %th Open issues - %th State - %th Due date + %th= _('Project') + %th= _('Open issues') + %th= _('State') + %th= _('Due date') %tr %td - project_name = group ? milestone.project.name : milestone.project.full_name @@ -33,8 +33,8 @@ = milestone.milestone.issues_visible_to_user(current_user).opened.count %td - if milestone.closed? - Closed + = _('Closed') - else - Open + = _('Open') %td = milestone.expires_at diff --git a/app/views/shared/profiles/_organization.html.haml b/app/views/shared/profiles/_organization.html.haml deleted file mode 100644 index 559b856a618..00000000000 --- a/app/views/shared/profiles/_organization.html.haml +++ /dev/null @@ -1 +0,0 @@ -= form.text_field :organization, label: s_('Profiles|Organization'), class: 'input-md', help: s_("Profiles|Who you represent or work for") diff --git a/changelogs/unreleased/208255-editing-markdown-fields-don-t-work.yml b/changelogs/unreleased/208255-editing-markdown-fields-don-t-work.yml new file mode 100644 index 00000000000..79425012b3f --- /dev/null +++ b/changelogs/unreleased/208255-editing-markdown-fields-don-t-work.yml @@ -0,0 +1,5 @@ +--- +title: Fix updating of Markdown fields when Markdown cache version is incremented +merge_request: 32219 +author: +type: fixed diff --git a/changelogs/unreleased/210018-remove-admin-ability-not-to-use-hashed-storage.yml b/changelogs/unreleased/210018-remove-admin-ability-not-to-use-hashed-storage.yml new file mode 100644 index 00000000000..19575ebb614 --- /dev/null +++ b/changelogs/unreleased/210018-remove-admin-ability-not-to-use-hashed-storage.yml @@ -0,0 +1,5 @@ +--- +title: Force hashed storage to always be enabled +merge_request: 31801 +author: +type: changed diff --git a/changelogs/unreleased/216505-public-dashboard-visibility.yml b/changelogs/unreleased/216505-public-dashboard-visibility.yml new file mode 100644 index 00000000000..ffbc0e7ad80 --- /dev/null +++ b/changelogs/unreleased/216505-public-dashboard-visibility.yml @@ -0,0 +1,5 @@ +--- +title: Fix public metrics dashboard visibility bug +merge_request: 31925 +author: +type: fixed diff --git a/changelogs/unreleased/22691-externelize-i18n-strings-from---app-views-shared-milestones-_top-ht.yml b/changelogs/unreleased/22691-externelize-i18n-strings-from---app-views-shared-milestones-_top-ht.yml new file mode 100644 index 00000000000..c373a4605c8 --- /dev/null +++ b/changelogs/unreleased/22691-externelize-i18n-strings-from---app-views-shared-milestones-_top-ht.yml @@ -0,0 +1,5 @@ +--- +title: Externalize i18n strings from ./app/views/shared/milestones/_top.html.haml +merge_request: 32148 +author: Gilang Gumilar +type: changed diff --git a/changelogs/unreleased/cl-update-indexer-230.yml b/changelogs/unreleased/cl-update-indexer-230.yml new file mode 100644 index 00000000000..da06321c464 --- /dev/null +++ b/changelogs/unreleased/cl-update-indexer-230.yml @@ -0,0 +1,5 @@ +--- +title: Update GitLab Elasticsearch Indexer to v2.3.0 +merge_request: 32199 +author: +type: other diff --git a/changelogs/unreleased/rc-remove_influxdb_supporting_libs.yml b/changelogs/unreleased/rc-remove_influxdb_supporting_libs.yml new file mode 100644 index 00000000000..f866e80989d --- /dev/null +++ b/changelogs/unreleased/rc-remove_influxdb_supporting_libs.yml @@ -0,0 +1,5 @@ +--- +title: Remove rake task `gitlab:track_deployment` +merge_request: 31404 +author: +type: removed diff --git a/changelogs/unreleased/update-deprecated-slot-syntax-in---app-assets-javascripts-confidential_me.yml b/changelogs/unreleased/update-deprecated-slot-syntax-in---app-assets-javascripts-confidential_me.yml new file mode 100644 index 00000000000..da481f4e582 --- /dev/null +++ b/changelogs/unreleased/update-deprecated-slot-syntax-in---app-assets-javascripts-confidential_me.yml @@ -0,0 +1,5 @@ +--- +title: Update deprecated slot syntax in ./app/assets/javascripts/confidential_merge_request/components/dropdown.vue +merge_request: 31999 +author: Gilang Gumilar +type: changed diff --git a/config/helpers/is_eslint.js b/config/helpers/is_eslint.js new file mode 100644 index 00000000000..5dfb7e533e4 --- /dev/null +++ b/config/helpers/is_eslint.js @@ -0,0 +1,18 @@ +/** + * Returns true if the given module is required from eslint + */ +const isESLint = mod => { + let parent = mod.parent; + + while (parent) { + if (parent.filename.includes('/eslint')) { + return true; + } + + parent = parent.parent; + } + + return false; +}; + +module.exports = isESLint; diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 3ad90ad7d65..addb2dd4f65 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -61,6 +61,8 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? Gitlab::Metrics::Samplers::PumaSampler.instance(Settings.monitoring.puma_sampler_interval).start end + Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION }, 1) + Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds rescue IOError => e Gitlab::ErrorTracking.track_exception(e) diff --git a/db/post_migrate/20200513171959_enable_hashed_storage.rb b/db/post_migrate/20200513171959_enable_hashed_storage.rb new file mode 100644 index 00000000000..53e52b1caff --- /dev/null +++ b/db/post_migrate/20200513171959_enable_hashed_storage.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class EnableHashedStorage < ActiveRecord::Migration[6.0] + DOWNTIME = false + + class ApplicationSetting < ActiveRecord::Base + self.table_name = 'application_settings' + end + + def up + ApplicationSetting.update_all(hashed_storage_enabled: true) + end + + def down + # in 13.0 we are forcing hashed storage to always be enabled for new projects + end +end diff --git a/db/structure.sql b/db/structure.sql index 2f28e4703e0..cc5c16772f8 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13823,6 +13823,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200512085150 20200512164334 20200513160930 +20200513171959 20200513234502 20200513235347 20200513235532 diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 696bc23c916..eee68c0da0a 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -196,27 +196,6 @@ production machine after installing the package, there should be no reason to re `rake gitlab:assets:compile` on the production machine. If you suspect that assets have been corrupted, you should reinstall the omnibus package. -## Tracking Deployments - -GitLab provides a Rake task that lets you track deployments in GitLab -Performance Monitoring. This Rake task simply stores the current GitLab version -in the GitLab Performance Monitoring database. - -To run `gitlab:track_deployment`: - -**Omnibus Installation** - -```shell -sudo gitlab-rake gitlab:track_deployment -``` - -**Source Installation** - -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:track_deployment RAILS_ENV=production -``` - ## Check TCP connectivity to a remote site Sometimes you need to know if your GitLab installation can connect to a TCP diff --git a/doc/api/settings.md b/doc/api/settings.md index d63e109cfe8..f63d126742a 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -262,7 +262,7 @@ are listed in the descriptions of the relevant settings. | `grafana_enabled` | boolean | no | Enable Grafana. | | `grafana_url` | string | no | Grafana URL. | | `gravatar_enabled` | boolean | no | Enable Gravatar. | -| `hashed_storage_enabled` | boolean | no | Create new projects using hashed storage paths: Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. (EXPERIMENTAL) | +| `hashed_storage_enabled` | boolean | no | Create new projects using hashed storage paths: Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. (Always enabled since 13.0, configuration will be removed in 14.0) | | `help_page_hide_commercial_content` | boolean | no | Hide marketing-related entries from help. | | `help_page_support_url` | string | no | Alternate support URL for help page and help dropdown. | | `help_page_text` | string | no | Custom text displayed on the help page. | diff --git a/doc/ci/jenkins/index.md b/doc/ci/jenkins/index.md index e43f0e4131c..c9235c62a84 100644 --- a/doc/ci/jenkins/index.md +++ b/doc/ci/jenkins/index.md @@ -40,6 +40,15 @@ things we have found that helps this: of the improvements that GitLab offers, and this requires (eventually) updating your implementation as part of the transition. +## JenkinsFile Wrapper + +We are building a [JenkinsFile Wrapper](https://gitlab.com/gitlab-org/jfr-container-builder/) which will allow +you to run a complete Jenkins instance inside of a GitLab job, including plugins. This can help ease the process +of transition, by letting you delay the migration of less urgent pipelines for a period of time. + +If you are interested, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) to +If you are interested, you might be able to [help GitLab test the wrapper](https://gitlab.com/gitlab-org/gitlab/-/issues/215675). + ## Important product differences There are some high level differences between the products worth mentioning: diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 42a686ed679..3e31a2169e2 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -394,6 +394,73 @@ Once you set them, they will be available for all subsequent pipelines. Any grou  +### Inherit environment variables + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22638) in GitLab 13.0. +> - It's deployed behind a feature flag (`ci_dependency_variables`), disabled by default. + +You can inherit environment variables from dependent jobs. + +This feature makes use of the [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) report feature. + +Example with [`dependencies`](../yaml/README.md#dependencies) keyword. + +```yaml +build: + stage: build + script: + - echo "BUILD_VERSION=hello" >> build.env + artifacts: + reports: + dotenv: build.env + +deploy: + stage: deploy + script: + - echo $BUILD_VERSION # => hello + dependencies: + - build +``` + +Example with the [`needs`](../yaml/README.md#artifact-downloads-with-needs) keyword: + +```yaml +build: + stage: build + script: + - echo "BUILD_VERSION=hello" >> build.env + artifacts: + reports: + dotenv: build.env + +deploy: + stage: deploy + script: + - echo $BUILD_VERSION # => hello + needs: + - job: build + artifacts: true +``` + +### Enable inherited environment variables **(CORE ONLY)** + +The Inherited Environment Variables feature is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:ci_dependency_variables) +``` + +To disable it: + +```ruby +Feature.disable(:ci_dependency_variables) +``` + ## Priority of environment variables Variables of different types can take precedence over other @@ -404,6 +471,7 @@ The order of precedence for variables is (from highest to lowest): 1. [Trigger variables](../triggers/README.md#making-use-of-trigger-variables) or [scheduled pipeline variables](../pipelines/schedules.md#using-variables). 1. Project-level [variables](#custom-environment-variables) or [protected variables](#protect-a-custom-variable). 1. Group-level [variables](#group-level-environment-variables) or [protected variables](#protect-a-custom-variable). +1. [Inherited environment variables](#inherit-environment-variables). 1. YAML-defined [job-level variables](../yaml/README.md#variables). 1. YAML-defined [global variables](../yaml/README.md#variables). 1. [Deployment variables](#deployment-environment-variables). diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 54f0bebc6db..d615b67f3d2 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -115,6 +115,9 @@ A limit of: GitLab offers Linux and Windows shared runners hosted on GitLab.com for executing your pipelines. +NOTE: **Note:** +Shared Runners provided by GitLab are **not** configurable. Consider [installing your own Runner](https://docs.gitlab.com/runner/install/) if you have specific configuration needs. + ### Linux Shared Runners Linux Shared Runners on GitLab.com run in [autoscale mode](https://docs.gitlab.com/runner/configuration/autoscale.html) and are powered by Google Cloud Platform. diff --git a/jest.config.base.js b/jest.config.base.js new file mode 100644 index 00000000000..1a1fd4e7b62 --- /dev/null +++ b/jest.config.base.js @@ -0,0 +1,92 @@ +const IS_EE = require('./config/helpers/is_ee_env'); +const isESLint = require('./config/helpers/is_eslint'); + +module.exports = path => { + const reporters = ['default']; + + // To have consistent date time parsing both in local and CI environments we set + // the timezone of the Node process. https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27738 + process.env.TZ = 'GMT'; + + if (process.env.CI) { + reporters.push([ + 'jest-junit', + { + output: './junit_jest.xml', + }, + ]); + } + + const glob = `${path}/**/*_spec.js`; + let testMatch = [`<rootDir>/${glob}`]; + if (IS_EE) { + testMatch.push(`<rootDir>/ee/${glob}`); + } + + // workaround for eslint-import-resolver-jest only resolving in test files + // see https://github.com/JoinColony/eslint-import-resolver-jest#note + if (isESLint(module)) { + testMatch = testMatch.map(path => path.replace('_spec.js', '')); + } + + const moduleNameMapper = { + '^~(/.*)$': '<rootDir>/app/assets/javascripts$1', + '^ee_component(/.*)$': + '<rootDir>/app/assets/javascripts/vue_shared/components/empty_component.js', + '^ee_else_ce(/.*)$': '<rootDir>/app/assets/javascripts$1', + '^helpers(/.*)$': '<rootDir>/spec/frontend/helpers$1', + '^vendor(/.*)$': '<rootDir>/vendor/assets/javascripts$1', + '\\.(jpg|jpeg|png|svg|css)$': '<rootDir>/spec/frontend/__mocks__/file_mock.js', + 'emojis(/.*).json': '<rootDir>/fixtures/emojis$1.json', + '^spec/test_constants$': '<rootDir>/spec/frontend/helpers/test_constants', + '^jest/(.*)$': '<rootDir>/spec/frontend/$1', + }; + + const collectCoverageFrom = ['<rootDir>/app/assets/javascripts/**/*.{js,vue}']; + + if (IS_EE) { + const rootDirEE = '<rootDir>/ee/app/assets/javascripts$1'; + Object.assign(moduleNameMapper, { + '^ee(/.*)$': rootDirEE, + '^ee_component(/.*)$': rootDirEE, + '^ee_else_ce(/.*)$': rootDirEE, + '^ee_jest/(.*)$': '<rootDir>/ee/spec/frontend/$1', + }); + + collectCoverageFrom.push(rootDirEE.replace('$1', '/**/*.{js,vue}')); + } + + const coverageDirectory = () => { + if (process.env.CI_NODE_INDEX && process.env.CI_NODE_TOTAL) { + return `<rootDir>/coverage-frontend/jest-${process.env.CI_NODE_INDEX}-${process.env.CI_NODE_TOTAL}`; + } + + return '<rootDir>/coverage-frontend/'; + }; + + return { + clearMocks: true, + testMatch, + moduleFileExtensions: ['js', 'json', 'vue'], + moduleNameMapper, + collectCoverageFrom, + coverageDirectory: coverageDirectory(), + coverageReporters: ['json', 'lcov', 'text-summary', 'clover'], + cacheDirectory: '<rootDir>/tmp/cache/jest', + modulePathIgnorePatterns: ['<rootDir>/.yarn-cache/'], + reporters, + setupFilesAfterEnv: ['<rootDir>/spec/frontend/test_setup.js', 'jest-canvas-mock'], + restoreMocks: true, + transform: { + '^.+\\.(gql|graphql)$': 'jest-transform-graphql', + '^.+\\.js$': 'babel-jest', + '^.+\\.vue$': 'vue-jest', + }, + transformIgnorePatterns: ['node_modules/(?!(@gitlab/ui|bootstrap-vue|three|monaco-editor)/)'], + timers: 'fake', + testEnvironment: '<rootDir>/spec/frontend/environment.js', + testEnvironmentOptions: { + IS_EE, + }, + }; +}; diff --git a/jest.config.integration.js b/jest.config.integration.js new file mode 100644 index 00000000000..573002c1a34 --- /dev/null +++ b/jest.config.integration.js @@ -0,0 +1,5 @@ +const baseConfig = require('./jest.config.base'); + +module.exports = { + ...baseConfig('spec/frontend_integration'), +}; diff --git a/jest.config.js b/jest.config.js deleted file mode 100644 index e9e1a598608..00000000000 --- a/jest.config.js +++ /dev/null @@ -1,103 +0,0 @@ -const IS_EE = require('./config/helpers/is_ee_env'); - -const reporters = ['default']; - -// To have consistent date time parsing both in local and CI environments we set -// the timezone of the Node process. https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27738 -process.env.TZ = 'GMT'; - -if (process.env.CI) { - reporters.push([ - 'jest-junit', - { - output: './junit_jest.xml', - }, - ]); -} - -let testMatch = ['<rootDir>/spec/frontend/**/*_spec.js']; -if (IS_EE) { - testMatch.push('<rootDir>/ee/spec/frontend/**/*_spec.js'); -} - -// workaround for eslint-import-resolver-jest only resolving in test files -// see https://github.com/JoinColony/eslint-import-resolver-jest#note -const { filename: parentModuleName } = module.parent; -const isESLint = parentModuleName && parentModuleName.includes('/eslint-import-resolver-jest/'); -if (isESLint) { - testMatch = testMatch.map(path => path.replace('_spec.js', '')); -} - -const moduleNameMapper = { - '^~(/.*)$': '<rootDir>/app/assets/javascripts$1', - '^ee_component(/.*)$': - '<rootDir>/app/assets/javascripts/vue_shared/components/empty_component.js', - '^ee_else_ce(/.*)$': '<rootDir>/app/assets/javascripts$1', - '^helpers(/.*)$': '<rootDir>/spec/frontend/helpers$1', - '^vendor(/.*)$': '<rootDir>/vendor/assets/javascripts$1', - '\\.(jpg|jpeg|png|svg|css)$': '<rootDir>/spec/frontend/__mocks__/file_mock.js', - 'emojis(/.*).json': '<rootDir>/fixtures/emojis$1.json', - '^spec/test_constants$': '<rootDir>/spec/frontend/helpers/test_constants', - '^jest/(.*)$': '<rootDir>/spec/frontend/$1', -}; - -const collectCoverageFrom = ['<rootDir>/app/assets/javascripts/**/*.{js,vue}']; - -if (IS_EE) { - const rootDirEE = '<rootDir>/ee/app/assets/javascripts$1'; - Object.assign(moduleNameMapper, { - '^ee(/.*)$': rootDirEE, - '^ee_component(/.*)$': rootDirEE, - '^ee_else_ce(/.*)$': rootDirEE, - '^ee_jest/(.*)$': '<rootDir>/ee/spec/frontend/$1', - }); - - collectCoverageFrom.push(rootDirEE.replace('$1', '/**/*.{js,vue}')); -} - -const coverageDirectory = () => { - if (process.env.CI_NODE_INDEX && process.env.CI_NODE_TOTAL) { - return `<rootDir>/coverage-frontend/jest-${process.env.CI_NODE_INDEX}-${process.env.CI_NODE_TOTAL}`; - } - - return '<rootDir>/coverage-frontend/'; -}; - -// eslint-disable-next-line import/no-commonjs -module.exports = { - clearMocks: true, - testMatch, - moduleFileExtensions: ['js', 'json', 'vue'], - moduleNameMapper, - collectCoverageFrom, - coverageDirectory: coverageDirectory(), - coverageReporters: ['json', 'lcov', 'text-summary', 'clover'], - cacheDirectory: '<rootDir>/tmp/cache/jest', - modulePathIgnorePatterns: ['<rootDir>/.yarn-cache/'], - reporters, - setupFilesAfterEnv: ['<rootDir>/spec/frontend/test_setup.js', 'jest-canvas-mock'], - restoreMocks: true, - transform: { - '^.+\\.(gql|graphql)$': 'jest-transform-graphql', - '^.+\\.js$': 'babel-jest', - '^.+\\.vue$': 'vue-jest', - }, - transformIgnorePatterns: ['node_modules/(?!(@gitlab/ui|bootstrap-vue|three|monaco-editor)/)'], - timers: 'fake', - testEnvironment: '<rootDir>/spec/frontend/environment.js', - testEnvironmentOptions: { - IS_EE, - }, -}; - -const karmaTestFile = process.argv.find(arg => arg.includes('spec/javascripts/')); -if (karmaTestFile) { - console.error(` -Files in spec/javascripts/ and ee/spec/javascripts need to be run with Karma. -Please use the following command instead: - -yarn karma -f ${karmaTestFile} - -`); - process.exit(1); -} diff --git a/jest.config.unit.js b/jest.config.unit.js new file mode 100644 index 00000000000..4627462c730 --- /dev/null +++ b/jest.config.unit.js @@ -0,0 +1,17 @@ +const baseConfig = require('./jest.config.base'); + +module.exports = { + ...baseConfig('spec/frontend'), +}; + +const karmaTestFile = process.argv.find(arg => arg.includes('spec/javascripts/')); +if (karmaTestFile) { + console.error(` +Files in spec/javascripts/ and ee/spec/javascripts need to be run with Karma. +Please use the following command instead: + +yarn karma -f ${karmaTestFile} + +`); + process.exit(1); +} diff --git a/lib/api/settings.rb b/lib/api/settings.rb index baae9513c98..e3a8f0671ef 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -185,6 +185,9 @@ module API attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 + attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) + attrs = filter_attributes_using_license(attrs) if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index c4f9407f7f0..57aeb6c4fb2 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -5,7 +5,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize include Gitlab::Routing - CODE_NAVIGATION_JOB_NAME = 'code_navigation' LATEST_COMMITS_LIMIT = 10 def initialize(project, commit_sha) @@ -17,7 +16,7 @@ module Gitlab return if Feature.disabled?(:code_navigation, project) return unless build - raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json") + raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) end private @@ -29,10 +28,11 @@ module Gitlab latest_commits_shas = project.repository.commits(commit_sha, limit: LATEST_COMMITS_LIMIT).map(&:sha) - artifact = ::Ci::JobArtifact - .for_sha(latest_commits_shas, project.id) - .for_job_name(CODE_NAVIGATION_JOB_NAME) - .last + artifact = + ::Ci::JobArtifact + .with_file_types(['lsif']) + .for_sha(latest_commits_shas, project.id) + .last artifact&.job end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb deleted file mode 100644 index 1f252572461..00000000000 --- a/lib/gitlab/metrics/influx_db.rb +++ /dev/null @@ -1,183 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module InfluxDb - extend ActiveSupport::Concern - include Gitlab::Metrics::Methods - - EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze - - MUTEX = Mutex.new - private_constant :MUTEX - - class_methods do - def influx_metrics_enabled? - settings[:enabled] || false - end - - # Prometheus histogram buckets used for arbitrary code measurements - - def settings - @settings ||= begin - current_settings = Gitlab::CurrentSettings.current_application_settings - - { - enabled: current_settings[:metrics_enabled], - pool_size: current_settings[:metrics_pool_size], - timeout: current_settings[:metrics_timeout], - method_call_threshold: current_settings[:metrics_method_call_threshold], - host: current_settings[:metrics_host], - port: current_settings[:metrics_port], - sample_interval: current_settings[:metrics_sample_interval] || 15, - packet_size: current_settings[:metrics_packet_size] || 1 - } - end - end - - def mri? - RUBY_ENGINE == 'ruby' - end - - def method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - # in milliseconds - @method_call_threshold ||= settings[:method_call_threshold] - end - - def submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool&.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - connection.write_points(slice) - rescue StandardError - end - end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - - def prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # UserFinder.new(some_username).find_by_username - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) - cpu_time = cpu_stop - cpu_start - - real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do - docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS - buckets EXECUTION_MEASUREMENT_BUCKETS - end - - real_duration_seconds.observe(trans.labels, real_time) - - cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do - docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS - buckets EXECUTION_MEASUREMENT_BUCKETS - with_feature "prometheus_metrics_measure_#{name}_cpu_duration" - end - cpu_duration_seconds.observe(trans.labels, cpu_time) - - # InfluxDB stores the _real_time and _cpu_time time values as milliseconds - trans.increment("#{name}_real_time", real_time.in_milliseconds, false) - trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) - trans.increment("#{name}_call_count", 1, false) - - retval - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def add_event(*args) - current_transaction&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def series_prefix - @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - def pool - if influx_metrics_enabled? - if @pool.nil? - MUTEX.synchronize do - @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client - .new(udp: { host: host, port: port }) - end - end - end - - @pool - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index d0c63a862c2..fbeda3b75e0 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -49,19 +49,6 @@ module Gitlab retval end - # Returns a Metric instance of the current method call. - def to_metric - Metric.new( - Instrumentation.series, - { - duration: real_time.in_milliseconds.to_i, - cpu_duration: cpu_time.in_milliseconds.to_i, - call_count: call_count - }, - method: @name - ) - end - # Returns true if the total runtime of this method exceeds the method call # threshold. def above_threshold? diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb deleted file mode 100644 index 30f181542be..00000000000 --- a/lib/gitlab/metrics/metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - # Class for storing details of a single metric (label, value, etc). - class Metric - JITTER_RANGE = (0.000001..0.001).freeze - - attr_reader :series, :values, :tags, :type - - # series - The name of the series (as a String) to store the metric in. - # values - A Hash containing the values to store. - # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}, type = :metric) - @values = values - @series = series - @tags = tags - @type = type - end - - def event? - type == :event - end - - # Returns a Hash in a format that can be directly written to InfluxDB. - def to_hash - # InfluxDB overwrites an existing point if a new point has the same - # series, tag set, and timestamp. In a highly concurrent environment - # this means that using the number of seconds since the Unix epoch is - # inevitably going to collide with another timestamp. For example, two - # Rails requests processed by different processes may end up generating - # metrics using the _exact_ same timestamp (in seconds). - # - # Due to the way InfluxDB is set up there's no solution to this problem, - # all we can do is lower the amount of collisions. We do this by using - # System.real_time which returns the nanoseconds as a Float providing - # greater accuracy. We then add a small random value that is large - # enough to distinguish most timestamps but small enough to not alter - # the timestamp significantly. - # - # See https://gitlab.com/gitlab-com/operations/issues/175 for more - # information. - time = System.real_time(:nanosecond) + rand(JITTER_RANGE) - - { - series: @series, - tags: @tags, - values: @values, - timestamp: time.to_i - } - end - end - end -end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb deleted file mode 100644 index b9eee3a15be..00000000000 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Samplers - # Class that sends certain metrics to InfluxDB at a specific interval. - # - # This class is used to gather statistics that can't be directly associated - # with a transaction such as system memory usage, garbage collection - # statistics, etc. - class InfluxSampler < BaseSampler - # interval - The sampling interval in seconds. - def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval]) - super(interval) - @last_step = nil - - @metrics = [] - end - - def sample - sample_memory_usage - sample_file_descriptors - - flush - ensure - @metrics.clear - end - - def flush - ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash)) - end - - def sample_memory_usage - add_metric('memory_usage', value: System.memory_usage_rss) - end - - def sample_file_descriptors - add_metric('file_descriptors', value: System.file_descriptor_count) - end - - def add_metric(series, values, tags = {}) - prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' - - @metrics << Metric.new("#{prefix}#{series}", values, tags) - end - end - end - end -end diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake deleted file mode 100644 index 6f101aea303..00000000000 --- a/lib/tasks/gitlab/track_deployment.rake +++ /dev/null @@ -1,9 +0,0 @@ -namespace :gitlab do - desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' - task track_deployment: :environment do - metric = Gitlab::Metrics::Metric - .new('deployments', version: Gitlab::VERSION) - - Gitlab::Metrics.submit_metrics([metric.to_hash]) - end -end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 41468d6c6c6..980ef7ef1f3 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -7416,6 +7416,9 @@ msgstr "" msgid "Destroy" msgstr "" +msgid "Detail" +msgstr "" + msgid "Details" msgstr "" @@ -9263,6 +9266,9 @@ msgstr "" msgid "FeatureFlags|Feature Flags" msgstr "" +msgid "FeatureFlags|Feature Flags will look different in the next milestone. No action is needed, but you may notice the functionality was changed to improve the workflow." +msgstr "" + msgid "FeatureFlags|Feature flag %{name} will be removed. Are you sure?" msgstr "" @@ -11039,6 +11045,9 @@ msgstr "" msgid "Hashed repository storage paths" msgstr "" +msgid "Hashed storage can't be disabled anymore for new projects" +msgstr "" + msgid "Have your users email" msgstr "" @@ -14986,6 +14995,9 @@ msgstr "" msgid "PackageRegistry|There are no %{packageType} packages yet" msgstr "" +msgid "PackageRegistry|There are no other versions of this package." +msgstr "" + msgid "PackageRegistry|There are no packages yet" msgstr "" @@ -14998,6 +15010,9 @@ msgstr "" msgid "PackageRegistry|To widen your search, change or remove the filters above." msgstr "" +msgid "PackageRegistry|Unable to fetch package version information." +msgstr "" + msgid "PackageRegistry|Unable to load package" msgstr "" @@ -20309,6 +20324,9 @@ msgstr "" msgid "Starts at (UTC)" msgstr "" +msgid "State" +msgstr "" + msgid "State your message to activate" msgstr "" @@ -23289,7 +23307,7 @@ msgstr "" msgid "Use hashed storage" msgstr "" -msgid "Use hashed storage paths for newly created and renamed repositories. Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Repository URL changes and may improve disk I/O performance." +msgid "Use hashed storage paths for newly created and renamed repositories. Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Repository URL changes and may improve disk I/O performance. (Always enabled since 13.0)" msgstr "" msgid "Use one line per URI" @@ -23745,6 +23763,9 @@ msgstr "" msgid "Version" msgstr "" +msgid "Versions" +msgstr "" + msgid "Very helpful" msgstr "" diff --git a/package.json b/package.json index 483e4e1eddc..5b5dddffc49 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,9 @@ "eslint-report": "eslint --max-warnings 0 --ext .js,.vue --format html --output-file ./eslint-report.html --no-inline-config .", "file-coverage": "scripts/frontend/file_test_coverage.js", "prejest": "yarn check-dependencies", - "jest": "jest", + "jest": "jest --config jest.config.unit.js", "jest-debug": "node --inspect-brk node_modules/.bin/jest --runInBand", + "jest:integration": "jest --config jest.config.integration.js", "jsdoc": "jsdoc -c config/jsdocs.config.js", "prekarma": "yarn check-dependencies", "karma": "BABEL_ENV=${BABEL_ENV:=karma} karma start --single-run true config/karma.config.js", diff --git a/spec/controllers/concerns/metrics_dashboard_spec.rb b/spec/controllers/concerns/metrics_dashboard_spec.rb index fd1f4fc1641..e2fa03670d9 100644 --- a/spec/controllers/concerns/metrics_dashboard_spec.rb +++ b/spec/controllers/concerns/metrics_dashboard_spec.rb @@ -45,7 +45,7 @@ describe MetricsDashboard do it 'returns the specified dashboard' do expect(json_response['dashboard']['dashboard']).to eq('Environment metrics') expect(json_response).not_to have_key('all_dashboards') - expect(json_response).not_to have_key('metrics_data') + expect(json_response).to have_key('metrics_data') end context 'when the params are in an alternate format' do @@ -54,7 +54,7 @@ describe MetricsDashboard do it 'returns the specified dashboard' do expect(json_response['dashboard']['dashboard']).to eq('Environment metrics') expect(json_response).not_to have_key('all_dashboards') - expect(json_response).not_to have_key('metrics_data') + expect(json_response).to have_key('metrics_data') end end diff --git a/spec/controllers/projects/grafana_api_controller_spec.rb b/spec/controllers/projects/grafana_api_controller_spec.rb index c62baa30fde..8502bd1ab0a 100644 --- a/spec/controllers/projects/grafana_api_controller_spec.rb +++ b/spec/controllers/projects/grafana_api_controller_spec.rb @@ -131,10 +131,11 @@ describe Projects::GrafanaApiController do get :metrics_dashboard, params: params expect(response).to have_gitlab_http_status(:ok) - expect(json_response).to eq({ + expect(json_response).to include({ 'dashboard' => '{}', 'status' => 'success' }) + expect(json_response).to include('metrics_data') end end diff --git a/spec/controllers/projects/prometheus/alerts_controller_spec.rb b/spec/controllers/projects/prometheus/alerts_controller_spec.rb index 451834e0962..e936cb5916e 100644 --- a/spec/controllers/projects/prometheus/alerts_controller_spec.rb +++ b/spec/controllers/projects/prometheus/alerts_controller_spec.rb @@ -352,7 +352,7 @@ describe Projects::Prometheus::AlertsController do get :metrics_dashboard, params: request_params(id: metric.id, environment_id: alert.environment.id), format: :json expect(response).to have_gitlab_http_status(:ok) - expect(json_response.keys).to contain_exactly('dashboard', 'status') + expect(json_response.keys).to contain_exactly('dashboard', 'status', 'metrics_data') end it 'is the correct embed' do diff --git a/spec/frontend/.eslintrc.yml b/spec/frontend/.eslintrc.yml index 8e6faa90c58..b9159191114 100644 --- a/spec/frontend/.eslintrc.yml +++ b/spec/frontend/.eslintrc.yml @@ -10,7 +10,7 @@ settings: - path import/resolver: jest: - jestConfigFile: 'jest.config.js' + jestConfigFile: 'jest.config.unit.js' globals: getJSONFixture: false loadFixtures: false diff --git a/spec/frontend_integration/.eslintrc.yml b/spec/frontend_integration/.eslintrc.yml new file mode 100644 index 00000000000..26b6f935ffb --- /dev/null +++ b/spec/frontend_integration/.eslintrc.yml @@ -0,0 +1,6 @@ +--- +extends: ../frontend/.eslintrc.yml +settings: + import/resolver: + jest: + jestConfigFile: 'jest.config.integration.js' diff --git a/spec/frontend_integration/README.md b/spec/frontend_integration/README.md new file mode 100644 index 00000000000..573a385d81e --- /dev/null +++ b/spec/frontend_integration/README.md @@ -0,0 +1,17 @@ +## Frontend Integration Specs + +This directory contains Frontend integration specs. Go to `spec/frontend` if you're looking for Frontend unit tests. + +Frontend integration specs: + +- Mock out the Backend. +- Don't test individual components, but instead test use cases. +- Are expected to run slower than unit tests. +- Could end up having their own environment. + +As a result, they deserve their own special place. + +## References + +- https://docs.gitlab.com/ee/development/testing_guide/testing_levels.html#frontend-integration-tests +- https://gitlab.com/gitlab-org/gitlab/-/issues/208800 diff --git a/spec/frontend_integration/ide/__snapshots__/ide_integration_spec.js.snap b/spec/frontend_integration/ide/__snapshots__/ide_integration_spec.js.snap new file mode 100644 index 00000000000..a76f7960d03 --- /dev/null +++ b/spec/frontend_integration/ide/__snapshots__/ide_integration_spec.js.snap @@ -0,0 +1,136 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`WebIDE runs 1`] = ` +<div> + <article + class="ide position-relative d-flex flex-column align-items-stretch" + > + <div + class="ide-view flex-grow d-flex" + > + <div + class="file-finder-overlay" + style="display: none;" + > + (jest: contents hidden) + </div> + <div + class="multi-file-commit-panel flex-column" + style="width: 340px;" + > + <div + class="multi-file-commit-panel-inner" + > + <div + class="multi-file-loading-container" + > + <div + class="animation-container" + > + <div + class="skeleton-line-1" + /> + <div + class="skeleton-line-2" + /> + <div + class="skeleton-line-3" + /> + </div> + </div> + <div + class="multi-file-loading-container" + > + <div + class="animation-container" + > + <div + class="skeleton-line-1" + /> + <div + class="skeleton-line-2" + /> + <div + class="skeleton-line-3" + /> + </div> + </div> + <div + class="multi-file-loading-container" + > + <div + class="animation-container" + > + <div + class="skeleton-line-1" + /> + <div + class="skeleton-line-2" + /> + <div + class="skeleton-line-3" + /> + </div> + </div> + </div> + <div + class="position-absolute position-top-0 position-bottom-0 drag-handle position-right-0" + size="340" + style="cursor: ew-resize;" + /> + </div> + <div + class="multi-file-edit-pane" + > + <div + class="ide-empty-state" + > + <div + class="row js-empty-state" + > + <div + class="col-12" + > + <div + class="svg-content svg-250" + > + <img + src="/test/empty_state.svg" + /> + </div> + </div> + <div + class="col-12" + > + <div + class="text-content text-center" + > + <h4> + Make and review changes in the browser with the Web IDE + </h4> + <div + class="gl-spinner-container" + > + <span + aria-hidden="true" + aria-label="Loading" + class="align-text-bottom gl-spinner gl-spinner-orange gl-spinner-md" + /> + </div> + </div> + </div> + </div> + </div> + </div> + </div> + <footer + class="ide-status-bar" + > + <div + class="ide-status-list d-flex ml-auto" + > + </div> + </footer> + </article> +</div> +`; diff --git a/spec/frontend_integration/ide/ide_integration_spec.js b/spec/frontend_integration/ide/ide_integration_spec.js new file mode 100644 index 00000000000..7e8fb3a32ee --- /dev/null +++ b/spec/frontend_integration/ide/ide_integration_spec.js @@ -0,0 +1,100 @@ +/** + * WARNING: WIP + * + * Please do not copy from this spec or use it as an example for anything. + * + * This is in place to iteratively set up the frontend integration testing environment + * and will be improved upon in a later iteration. + * + * See https://gitlab.com/gitlab-org/gitlab/-/issues/208800 for more information. + */ +import MockAdapter from 'axios-mock-adapter'; +import axios from '~/lib/utils/axios_utils'; +import { initIde } from '~/ide'; + +jest.mock('~/api', () => { + return { + project: jest.fn().mockImplementation(() => new Promise(() => {})), + }; +}); + +jest.mock('~/ide/services/gql', () => { + return { + query: jest.fn().mockImplementation(() => new Promise(() => {})), + }; +}); + +describe('WebIDE', () => { + let vm; + let root; + let mock; + let initData; + let location; + + beforeEach(() => { + root = document.createElement('div'); + initData = { + emptyStateSvgPath: '/test/empty_state.svg', + noChangesStateSvgPath: '/test/no_changes_state.svg', + committedStateSvgPath: '/test/committed_state.svg', + pipelinesEmptyStateSvgPath: '/test/pipelines_empty_state.svg', + promotionSvgPath: '/test/promotion.svg', + ciHelpPagePath: '/test/ci_help_page', + webIDEHelpPagePath: '/test/web_ide_help_page', + clientsidePreviewEnabled: 'true', + renderWhitespaceInCode: 'false', + codesandboxBundlerUrl: 'test/codesandbox_bundler', + }; + + mock = new MockAdapter(axios); + mock.onAny('*').reply(() => new Promise(() => {})); + + location = { pathname: '/-/ide/project/gitlab-test/test', search: '', hash: '' }; + Object.defineProperty(window, 'location', { + get() { + return location; + }, + }); + }); + + afterEach(() => { + vm.$destroy(); + vm = null; + + mock.restore(); + }); + + const createComponent = () => { + const el = document.createElement('div'); + Object.assign(el.dataset, initData); + root.appendChild(el); + vm = initIde(el); + }; + + expect.addSnapshotSerializer({ + test(value) { + return value instanceof HTMLElement && !value.$_hit; + }, + print(element, serialize) { + element.$_hit = true; + element.querySelectorAll('[style]').forEach(el => { + el.$_hit = true; + if (el.style.display === 'none') { + el.textContent = '(jest: contents hidden)'; + } + }); + + return serialize(element) + .replace(/^\s*<!---->$/gm, '') + .replace(/\n\s*\n/gm, '\n'); + }, + }); + + it('runs', () => { + createComponent(); + + return vm.$nextTick().then(() => { + expect(root).toMatchSnapshot(); + }); + }); +}); diff --git a/spec/lib/gitlab/code_navigation_path_spec.rb b/spec/lib/gitlab/code_navigation_path_spec.rb index f9f335ff48d..938a2f821fd 100644 --- a/spec/lib/gitlab/code_navigation_path_spec.rb +++ b/spec/lib/gitlab/code_navigation_path_spec.rb @@ -6,9 +6,8 @@ describe Gitlab::CodeNavigationPath do context 'when there is an artifact with code navigation data' do let_it_be(:project) { create(:project, :repository) } let_it_be(:sha) { project.repository.commits('master', limit: 5).last.id } - let_it_be(:build_name) { Gitlab::CodeNavigationPath::CODE_NAVIGATION_JOB_NAME } let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: sha) } - let_it_be(:job) { create(:ci_build, pipeline: pipeline, name: build_name) } + let_it_be(:job) { create(:ci_build, pipeline: pipeline) } let_it_be(:artifact) { create(:ci_job_artifact, :lsif, job: job) } let(:commit_sha) { sha } @@ -18,7 +17,7 @@ describe Gitlab::CodeNavigationPath do context 'when a pipeline exist for a sha' do it 'returns path to a file in the artifact' do - expect(subject).to eq("/#{project.full_path}/-/jobs/#{job.id}/artifacts/raw/lsif/#{path}.json") + expect(subject).to eq("/#{project.full_path}/-/jobs/#{job.id}/artifacts/raw/lsif/#{path}.json?file_type=lsif") end end @@ -26,7 +25,7 @@ describe Gitlab::CodeNavigationPath do let(:commit_sha) { project.commit.id } it 'returns path to a file in the artifact' do - expect(subject).to eq("/#{project.full_path}/-/jobs/#{job.id}/artifacts/raw/lsif/#{path}.json") + expect(subject).to eq("/#{project.full_path}/-/jobs/#{job.id}/artifacts/raw/lsif/#{path}.json?file_type=lsif") end end diff --git a/spec/lib/gitlab/metrics/method_call_spec.rb b/spec/lib/gitlab/metrics/method_call_spec.rb index 3b5e04e2df5..229db67ec88 100644 --- a/spec/lib/gitlab/metrics/method_call_spec.rb +++ b/spec/lib/gitlab/metrics/method_call_spec.rb @@ -76,25 +76,6 @@ describe Gitlab::Metrics::MethodCall do end end - describe '#to_metric' do - it 'returns a Metric instance' do - expect(method_call).to receive(:real_time).and_return(4.0001).twice - expect(method_call).to receive(:cpu_time).and_return(3.0001) - - method_call.measure { 'foo' } - metric = method_call.to_metric - - expect(metric).to be_an_instance_of(Gitlab::Metrics::Metric) - expect(metric.series).to eq('rails_method_calls') - - expect(metric.values[:duration]).to eq(4000) - expect(metric.values[:cpu_duration]).to eq(3000) - expect(metric.values[:call_count]).to be_an(Integer) - - expect(metric.tags).to eq({ method: 'Foo#bar' }) - end - end - describe '#above_threshold?' do before do allow(Gitlab::Metrics).to receive(:method_call_threshold).and_return(100) diff --git a/spec/lib/gitlab/metrics/metric_spec.rb b/spec/lib/gitlab/metrics/metric_spec.rb deleted file mode 100644 index 611b59231ba..00000000000 --- a/spec/lib/gitlab/metrics/metric_spec.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::Metrics::Metric do - let(:metric) do - described_class.new('foo', { number: 10 }, { host: 'localtoast' }) - end - - describe '#series' do - subject { metric.series } - - it { is_expected.to eq('foo') } - end - - describe '#values' do - subject { metric.values } - - it { is_expected.to eq({ number: 10 }) } - end - - describe '#tags' do - subject { metric.tags } - - it { is_expected.to eq({ host: 'localtoast' }) } - end - - describe '#type' do - subject { metric.type } - - it { is_expected.to eq(:metric) } - end - - describe '#event?' do - it 'returns false for a regular metric' do - expect(metric.event?).to eq(false) - end - - it 'returns true for an event metric' do - expect(metric).to receive(:type).and_return(:event) - - expect(metric.event?).to eq(true) - end - end - - describe '#to_hash' do - it 'returns a Hash' do - expect(metric.to_hash).to be_an_instance_of(Hash) - end - - describe 'the returned Hash' do - let(:hash) { metric.to_hash } - - it 'includes the series' do - expect(hash[:series]).to eq('foo') - end - - it 'includes the tags' do - expect(hash[:tags]).to be_an_instance_of(Hash) - end - - it 'includes the values' do - expect(hash[:values]).to eq({ number: 10 }) - end - - it 'includes the timestamp' do - expect(hash[:timestamp]).to be_an(Integer) - end - end - end -end diff --git a/spec/lib/gitlab/metrics/samplers/influx_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/influx_sampler_spec.rb deleted file mode 100644 index 3e0b886be55..00000000000 --- a/spec/lib/gitlab/metrics/samplers/influx_sampler_spec.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Gitlab::Metrics::Samplers::InfluxSampler do - let(:sampler) { described_class.new(5) } - - describe '#start' do - it 'runs once and gathers a sample at a given interval' do - expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice - expect(sampler).to receive(:sample).once - expect(sampler).to receive(:running).and_return(true, false) - - sampler.start.join - end - end - - describe '#sample' do - it 'samples various statistics' do - expect(sampler).to receive(:sample_memory_usage) - expect(sampler).to receive(:sample_file_descriptors) - expect(sampler).to receive(:flush) - - sampler.sample - end - end - - describe '#flush' do - it 'schedules the metrics using Sidekiq' do - expect(Gitlab::Metrics).to receive(:submit_metrics) - .with([an_instance_of(Hash)]) - - sampler.sample_memory_usage - sampler.flush - end - end - - describe '#sample_memory_usage' do - it 'adds a metric containing the memory usage' do - expect(Gitlab::Metrics::System).to receive(:memory_usage_rss) - .and_return(9000) - - expect(sampler).to receive(:add_metric) - .with(/memory_usage/, value: 9000) - .and_call_original - - sampler.sample_memory_usage - end - end - - describe '#sample_file_descriptors' do - it 'adds a metric containing the amount of open file descriptors' do - expect(Gitlab::Metrics::System).to receive(:file_descriptor_count) - .and_return(4) - - expect(sampler).to receive(:add_metric) - .with(/file_descriptors/, value: 4) - .and_call_original - - sampler.sample_file_descriptors - end - end - - describe '#add_metric' do - it 'prefixes the series name for a Rails process' do - expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(false) - - expect(Gitlab::Metrics::Metric).to receive(:new) - .with('rails_cats', { value: 10 }, {}) - .and_call_original - - sampler.add_metric('cats', value: 10) - end - - it 'prefixes the series name for a Sidekiq process' do - expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(true) - - expect(Gitlab::Metrics::Metric).to receive(:new) - .with('sidekiq_cats', { value: 10 }, {}) - .and_call_original - - sampler.add_metric('cats', value: 10) - end - end - - describe '#sleep_interval' do - it 'returns a Numeric' do - expect(sampler.sleep_interval).to be_a_kind_of(Numeric) - end - - # Testing random behaviour is very hard, so treat this test as a basic smoke - # test instead of a very accurate behaviour/unit test. - it 'does not return the same interval twice in a row' do - last = nil - - 100.times do - interval = sampler.sleep_interval - - expect(interval).not_to eq(last) - - last = interval - end - end - end -end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index c552f8e1c42..64308af38f9 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -103,6 +103,8 @@ describe ApplicationSetting do it { is_expected.not_to allow_value(10.5).for(:raw_blob_request_limit) } it { is_expected.not_to allow_value(-1).for(:raw_blob_request_limit) } + it { is_expected.not_to allow_value(false).for(:hashed_storage_enabled) } + context 'grafana_url validations' do before do subject.instance_variable_set(:@parsed_grafana_url, nil) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 2a28ac49672..8b68840c889 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2375,12 +2375,14 @@ describe Ci::Build do let(:pipeline_pre_var) { { key: 'pipeline', value: 'value', public: true, masked: false } } let(:build_yaml_var) { { key: 'yaml', value: 'value', public: true, masked: false } } let(:job_jwt_var) { { key: 'CI_JOB_JWT', value: 'ci.job.jwt', public: false, masked: true } } + let(:job_dependency_var) { { key: 'job_dependency', value: 'value', public: true, masked: false } } before do allow(build).to receive(:predefined_variables) { [build_pre_var] } allow(build).to receive(:yaml_variables) { [build_yaml_var] } allow(build).to receive(:persisted_variables) { [] } allow(build).to receive(:job_jwt_variables) { [job_jwt_var] } + allow(build).to receive(:dependency_variables) { [job_dependency_var] } allow_any_instance_of(Project) .to receive(:predefined_variables) { [project_pre_var] } @@ -2398,6 +2400,7 @@ describe Ci::Build do project_pre_var, pipeline_pre_var, build_yaml_var, + job_dependency_var, { key: 'secret', value: 'value', public: false, masked: false }]) end end @@ -3008,6 +3011,15 @@ describe Ci::Build do end end end + + context 'when build has dependency which has dotenv variable' do + let!(:prepare) { create(:ci_build, pipeline: pipeline, stage_idx: 0) } + let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: [prepare.name] }) } + + let!(:job_variable) { create(:ci_job_variable, :dotenv_source, job: prepare) } + + it { is_expected.to include(key: job_variable.key, value: job_variable.value, public: false, masked: false) } + end end describe '#scoped_variables' do @@ -3070,6 +3082,33 @@ describe Ci::Build do end end end + + context 'with dependency variables' do + let!(:prepare) { create(:ci_build, name: 'prepare', pipeline: pipeline, stage_idx: 0) } + let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['prepare'] }) } + + let!(:job_variable) { create(:ci_job_variable, :dotenv_source, job: prepare) } + + context 'FF ci_dependency_variables is enabled' do + before do + stub_feature_flags(ci_dependency_variables: true) + end + + it 'inherits dependent variables' do + expect(build.scoped_variables.to_hash).to include(job_variable.key => job_variable.value) + end + end + + context 'FF ci_dependency_variables is disabled' do + before do + stub_feature_flags(ci_dependency_variables: false) + end + + it 'does not inherit dependent variables' do + expect(build.scoped_variables.to_hash).not_to include(job_variable.key => job_variable.value) + end + end + end end describe '#secret_group_variables' do @@ -3314,6 +3353,41 @@ describe Ci::Build do end end + describe '#dependency_variables' do + subject { build.dependency_variables } + + context 'when using dependencies' do + let!(:prepare1) { create(:ci_build, name: 'prepare1', pipeline: pipeline, stage_idx: 0) } + let!(:prepare2) { create(:ci_build, name: 'prepare2', pipeline: pipeline, stage_idx: 0) } + let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['prepare1'] }) } + + let!(:job_variable_1) { create(:ci_job_variable, :dotenv_source, job: prepare1) } + let!(:job_variable_2) { create(:ci_job_variable, job: prepare1) } + let!(:job_variable_3) { create(:ci_job_variable, :dotenv_source, job: prepare2) } + + it 'inherits only dependent variables' do + expect(subject.to_hash).to eq(job_variable_1.key => job_variable_1.value) + end + end + + context 'when using needs' do + let!(:prepare1) { create(:ci_build, name: 'prepare1', pipeline: pipeline, stage_idx: 0) } + let!(:prepare2) { create(:ci_build, name: 'prepare2', pipeline: pipeline, stage_idx: 0) } + let!(:prepare3) { create(:ci_build, name: 'prepare3', pipeline: pipeline, stage_idx: 0) } + let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, scheduling_type: 'dag') } + let!(:build_needs_prepare1) { create(:ci_build_need, build: build, name: 'prepare1', artifacts: true) } + let!(:build_needs_prepare2) { create(:ci_build_need, build: build, name: 'prepare2', artifacts: false) } + + let!(:job_variable_1) { create(:ci_job_variable, :dotenv_source, job: prepare1) } + let!(:job_variable_2) { create(:ci_job_variable, :dotenv_source, job: prepare2) } + let!(:job_variable_3) { create(:ci_job_variable, :dotenv_source, job: prepare3) } + + it 'inherits only needs with artifacts variables' do + expect(subject.to_hash).to eq(job_variable_1.key => job_variable_1.value) + end + end + end + describe 'state transition: any => [:preparing]' do let(:build) { create(:ci_build, :created) } diff --git a/spec/models/concerns/cache_markdown_field_spec.rb b/spec/models/concerns/cache_markdown_field_spec.rb index 697a9e98505..193144fcb0e 100644 --- a/spec/models/concerns/cache_markdown_field_spec.rb +++ b/spec/models/concerns/cache_markdown_field_spec.rb @@ -223,6 +223,10 @@ describe CacheMarkdownField, :clean_gitlab_redis_cache do end context 'when the markdown cache is up to date' do + before do + thing.try(:save) + end + it 'does not call #refresh_markdown_cache' do expect(thing).not_to receive(:refresh_markdown_cache) @@ -256,6 +260,54 @@ describe CacheMarkdownField, :clean_gitlab_redis_cache do let(:klass) { ar_class } it_behaves_like 'a class with cached markdown fields' + + describe '#attribute_invalidated?' do + let(:thing) { klass.create(description: markdown, description_html: html, cached_markdown_version: cache_version) } + + it 'returns true when cached_markdown_version is different' do + thing.cached_markdown_version += 1 + + expect(thing.attribute_invalidated?(:description_html)).to eq(true) + end + + it 'returns true when markdown is changed' do + thing.description = updated_markdown + + expect(thing.attribute_invalidated?(:description_html)).to eq(true) + end + + it 'returns true when both markdown and HTML are changed' do + thing.description = updated_markdown + thing.description_html = updated_html + + expect(thing.attribute_invalidated?(:description_html)).to eq(true) + end + + it 'returns false when there are no changes' do + expect(thing.attribute_invalidated?(:description_html)).to eq(false) + end + end + + context 'when cache version is updated' do + let(:old_version) { cache_version - 1 } + let(:old_html) { '<p data-sourcepos="1:1-1:5" dir="auto" class="some-old-class"><code>Foo</code></p>' } + + let(:thing) do + # This forces the record to have outdated HTML. We can't use `create` because the `before_create` hook + # would re-render the HTML to the latest version + klass.create.tap do |thing| + thing.update_columns(description: markdown, description_html: old_html, cached_markdown_version: old_version) + end + end + + it 'correctly updates cached HTML even if refresh_markdown_cache is called before updating the attribute' do + thing.refresh_markdown_cache + + thing.update(description: updated_markdown) + + expect(thing.description_html).to eq(updated_html) + end + end end context 'for other classes' do diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index bbe769c6973..ff99661b41e 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -218,16 +218,41 @@ describe ProjectPolicy do project.project_feature.update(builds_access_level: ProjectFeature::DISABLED) end - it 'disallows all permissions except pipeline when the feature is disabled' do - builds_permissions = [ - :create_build, :read_build, :update_build, :admin_build, :destroy_build, - :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, - :create_environment, :read_environment, :update_environment, :admin_environment, :destroy_environment, - :create_cluster, :read_cluster, :update_cluster, :admin_cluster, :destroy_cluster, - :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment - ] + context 'without metrics_dashboard_allowed' do + before do + project.project_feature.update(metrics_dashboard_access_level: ProjectFeature::DISABLED) + end - expect_disallowed(*builds_permissions) + it 'disallows all permissions except pipeline when the feature is disabled' do + builds_permissions = [ + :create_build, :read_build, :update_build, :admin_build, :destroy_build, + :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, + :create_environment, :read_environment, :update_environment, :admin_environment, :destroy_environment, + :create_cluster, :read_cluster, :update_cluster, :admin_cluster, :destroy_cluster, + :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment + ] + + expect_disallowed(*builds_permissions) + end + end + + context 'with metrics_dashboard_allowed' do + before do + project.project_feature.update(metrics_dashboard_access_level: ProjectFeature::ENABLED) + end + + it 'disallows all permissions except pipeline and read_environment when the feature is disabled' do + builds_permissions = [ + :create_build, :read_build, :update_build, :admin_build, :destroy_build, + :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, + :create_environment, :update_environment, :admin_environment, :destroy_environment, + :create_cluster, :read_cluster, :update_cluster, :admin_cluster, :destroy_cluster, + :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment + ] + + expect_disallowed(*builds_permissions) + expect_allowed(:read_environment) + end end end @@ -252,20 +277,49 @@ describe ProjectPolicy do context 'repository feature' do subject { described_class.new(owner, project) } - it 'disallows all permissions when the feature is disabled' do + before do project.project_feature.update(repository_access_level: ProjectFeature::DISABLED) + end - repository_permissions = [ - :create_pipeline, :update_pipeline, :admin_pipeline, :destroy_pipeline, - :create_build, :read_build, :update_build, :admin_build, :destroy_build, - :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, - :create_environment, :read_environment, :update_environment, :admin_environment, :destroy_environment, - :create_cluster, :read_cluster, :update_cluster, :admin_cluster, - :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment, - :destroy_release - ] + context 'without metrics_dashboard_allowed' do + before do + project.project_feature.update(metrics_dashboard_access_level: ProjectFeature::DISABLED) + end - expect_disallowed(*repository_permissions) + it 'disallows all permissions when the feature is disabled' do + repository_permissions = [ + :create_pipeline, :update_pipeline, :admin_pipeline, :destroy_pipeline, + :create_build, :read_build, :update_build, :admin_build, :destroy_build, + :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, + :create_environment, :read_environment, :update_environment, :admin_environment, :destroy_environment, + :create_cluster, :read_cluster, :update_cluster, :admin_cluster, + :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment, + :destroy_release + ] + + expect_disallowed(*repository_permissions) + end + end + + context 'with metrics_dashboard_allowed' do + before do + project.project_feature.update(metrics_dashboard_access_level: ProjectFeature::ENABLED) + end + + it 'disallows all permissions when the feature is disabled' do + repository_permissions = [ + :create_pipeline, :update_pipeline, :admin_pipeline, :destroy_pipeline, + :create_build, :read_build, :update_build, :admin_build, :destroy_build, + :create_pipeline_schedule, :read_pipeline_schedule, :update_pipeline_schedule, :admin_pipeline_schedule, :destroy_pipeline_schedule, + :create_environment, :update_environment, :admin_environment, :destroy_environment, + :create_cluster, :read_cluster, :update_cluster, :admin_cluster, + :create_deployment, :read_deployment, :update_deployment, :admin_deployment, :destroy_deployment, + :destroy_release + ] + + expect_disallowed(*repository_permissions) + expect_allowed(:read_environment) + end end end @@ -576,8 +630,8 @@ describe ProjectPolicy do it { is_expected.to be_allowed(:metrics_dashboard) } it { is_expected.to be_allowed(:read_prometheus) } it { is_expected.to be_allowed(:read_deployment) } - it { is_expected.to be_allowed(:read_metrics_user_starred_dashboard) } - it { is_expected.to be_allowed(:create_metrics_user_starred_dashboard) } + it { is_expected.to be_disallowed(:read_metrics_user_starred_dashboard) } + it { is_expected.to be_disallowed(:create_metrics_user_starred_dashboard) } end end end diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index b071a5435a8..a5b95bc59a5 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -159,6 +159,14 @@ describe API::Settings, 'Settings' do expect(json_response['allow_local_requests_from_hooks_and_services']).to eq(true) end + it 'disables ability to switch to legacy storage' do + put api("/application/settings", admin), + params: { hashed_storage_enabled: false } + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response['hashed_storage_enabled']).to eq(true) + end + context 'external policy classification settings' do let(:settings) do { |