diff options
78 files changed, 870 insertions, 301 deletions
diff --git a/app/assets/javascripts/cycle_analytics/cycle_analytics_service.js b/app/assets/javascripts/cycle_analytics/cycle_analytics_service.js index a0426301a0a..babbfe93082 100644 --- a/app/assets/javascripts/cycle_analytics/cycle_analytics_service.js +++ b/app/assets/javascripts/cycle_analytics/cycle_analytics_service.js @@ -8,22 +8,26 @@ export default class CycleAnalyticsService { } fetchCycleAnalyticsData(options = { startDate: 30 }) { + const { startDate, projectIds } = options; + return this.axios .get('', { params: { - 'cycle_analytics[start_date]': options.startDate, + 'cycle_analytics[start_date]': startDate, + 'cycle_analytics[project_ids]': projectIds, }, }) .then(x => x.data); } fetchStageData(options) { - const { stage, startDate } = options; + const { stage, startDate, projectIds } = options; return this.axios .get(`events/${stage.name}.json`, { params: { 'cycle_analytics[start_date]': startDate, + 'cycle_analytics[project_ids]': projectIds, }, }) .then(x => x.data); diff --git a/app/assets/javascripts/performance_bar/components/performance_bar_app.vue b/app/assets/javascripts/performance_bar/components/performance_bar_app.vue index 015c1527500..f05db8376a4 100644 --- a/app/assets/javascripts/performance_bar/components/performance_bar_app.vue +++ b/app/assets/javascripts/performance_bar/components/performance_bar_app.vue @@ -42,6 +42,12 @@ export default { keys: ['feature', 'request'], }, { + metric: 'rugged', + header: 'Rugged calls', + details: 'details', + keys: ['feature', 'args'], + }, + { metric: 'redis', header: 'Redis calls', details: 'details', diff --git a/app/assets/stylesheets/framework/tooltips.scss b/app/assets/stylesheets/framework/tooltips.scss index 98f28987a82..edc2fb532c8 100644 --- a/app/assets/stylesheets/framework/tooltips.scss +++ b/app/assets/stylesheets/framework/tooltips.scss @@ -1,7 +1,6 @@ .tooltip-inner { - font-size: $tooltip-font-size; + font-size: $gl-font-size-small; border-radius: $border-radius-default; - line-height: 16px; + line-height: $gl-line-height; font-weight: $gl-font-weight-normal; - padding: 8px; } diff --git a/app/assets/stylesheets/framework/variables_overrides.scss b/app/assets/stylesheets/framework/variables_overrides.scss index ea96381a098..604b48e11ab 100644 --- a/app/assets/stylesheets/framework/variables_overrides.scss +++ b/app/assets/stylesheets/framework/variables_overrides.scss @@ -48,3 +48,7 @@ $spacers: ( 9: ($spacer * 8) ); $pagination-color: $gl-text-color; +$tooltip-padding-y: 0.5rem; +$tooltip-padding-x: 0.75rem; +$tooltip-arrow-height: 0.5rem; +$tooltip-arrow-width: 1rem; diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index aa4aa0fbdac..ebb50fc8b10 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -10,7 +10,7 @@ class Import::GithubController < Import::BaseController rescue_from Octokit::Unauthorized, with: :provider_unauthorized def new - if github_import_configured? && logged_in_with_provider? + if !ci_cd_only? && github_import_configured? && logged_in_with_provider? go_to_provider_for_permissions elsif session[access_token_key] redirect_to status_import_url @@ -169,11 +169,15 @@ class Import::GithubController < Import::BaseController # rubocop: enable CodeReuse/ActiveRecord def provider_auth - if session[access_token_key].blank? + if !ci_cd_only? && session[access_token_key].blank? go_to_provider_for_permissions end end + def ci_cd_only? + %w[1 true].include?(params[:ci_cd_only]) + end + def client_options {} end diff --git a/app/views/layouts/nav/sidebar/_group.html.haml b/app/views/layouts/nav/sidebar/_group.html.haml index 4b5ccc33716..48c9f19f89f 100644 --- a/app/views/layouts/nav/sidebar/_group.html.haml +++ b/app/views/layouts/nav/sidebar/_group.html.haml @@ -36,8 +36,6 @@ %span = _('Activity') - = render_if_exists 'groups/sidebar/security_dashboard' # EE-specific - - if group_sidebar_link?(:contribution_analytics) = nav_link(path: 'analytics#show') do = link_to group_analytics_path(@group), title: _('Contribution Analytics'), data: { placement: 'right' } do @@ -105,6 +103,8 @@ = _('Merge Requests') %span.badge.badge-pill.count.merge_counter.js-merge-counter.fly-out-badge= number_with_delimiter(merge_requests_count) + = render_if_exists "layouts/nav/ee/security_link" # EE-specific + - if group_sidebar_link?(:kubernetes) = nav_link(controller: [:clusters]) do = link_to group_clusters_path(@group) do diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml index 8caae5eed83..d1634eb62c0 100644 --- a/app/views/layouts/nav/sidebar/_project.html.haml +++ b/app/views/layouts/nav/sidebar/_project.html.haml @@ -34,10 +34,6 @@ = link_to project_releases_path(@project), title: _('Releases'), class: 'shortcuts-project-releases' do %span= _('Releases') - = render_if_exists 'projects/sidebar/security_dashboard' - - = render_if_exists 'projects/sidebar/dependencies' - - if can?(current_user, :read_cycle_analytics, @project) = nav_link(path: 'cycle_analytics#show') do = link_to project_cycle_analytics_path(@project), title: _('Cycle Analytics'), class: 'shortcuts-project-cycle-analytics' do @@ -203,6 +199,8 @@ %span = _('Charts') + = render_if_exists 'layouts/nav/sidebar/project_security_link' # EE-specific + - if project_nav_tab? :operations = nav_link(controller: sidebar_operations_paths) do = link_to sidebar_operations_link_path, class: 'shortcuts-operations qa-link-operations' do diff --git a/changelogs/unreleased/11090-export-design-management-lfs-data.yml b/changelogs/unreleased/11090-export-design-management-lfs-data.yml new file mode 100644 index 00000000000..36b773124d7 --- /dev/null +++ b/changelogs/unreleased/11090-export-design-management-lfs-data.yml @@ -0,0 +1,5 @@ +--- +title: Add support for exporting repository type data for LFS objects +merge_request: 30830 +author: +type: changed diff --git a/changelogs/unreleased/ab-count-strategies.yml b/changelogs/unreleased/ab-count-strategies.yml new file mode 100644 index 00000000000..bd95ff45d6f --- /dev/null +++ b/changelogs/unreleased/ab-count-strategies.yml @@ -0,0 +1,5 @@ +--- +title: Use tablesample approximate counting by default. +merge_request: 31048 +author: +type: performance diff --git a/changelogs/unreleased/sh-add-rugged-to-peek.yml b/changelogs/unreleased/sh-add-rugged-to-peek.yml new file mode 100644 index 00000000000..8a030f3daf2 --- /dev/null +++ b/changelogs/unreleased/sh-add-rugged-to-peek.yml @@ -0,0 +1,5 @@ +--- +title: Add Rugged calls to performance bar +merge_request: 30983 +author: +type: other diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index 1ffd133239d..8416ae430c7 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -26,6 +26,7 @@ Peek.into PEEK_DB_VIEW Peek.into Peek::Views::Gitaly Peek.into Peek::Views::Rblineprof Peek.into Peek::Views::RedisDetailed +Peek.into Peek::Views::Rugged Peek.into Peek::Views::GC Peek.into Peek::Views::Tracing if Labkit::Tracing.tracing_url_enabled? diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 0e11dffa0d6..fd076bb79d8 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -17,7 +17,7 @@ You are encouraged to first read through all the steps before executing them in your testing/production environment. NOTE: **Note:** -**Do not** set up any custom authentication for the **secondary** nodes. This will be handled by the **primary** node. +**Do not** set up any custom authentication for the **secondary** nodes. This will be handled by the **primary** node. Any change that requires access to the **Admin Area** needs to be done in the **primary** node because the **secondary** node is a read-only replica. @@ -242,7 +242,7 @@ node's Geo Nodes dashboard in your browser.  If your installation isn't working properly, check the -[troubleshooting document]. +[troubleshooting document](troubleshooting.md). The two most obvious issues that can become apparent in the dashboard are: diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index 166ee94eca4..550b3b07a95 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -251,7 +251,7 @@ Omnibus is the following: 1. Check the steps about defining `postgresql['sql_user_password']`, `gitlab_rails['db_password']`. 1. Make sure `postgresql['max_replication_slots']` matches the number of **secondary** Geo nodes locations. 1. Install GitLab on the **secondary** server. -1. Re-run the [database replication process][database-replication]. +1. Re-run the [database replication process](database.md#step-3-initiate-the-replication-process). ## Special update notes for 9.0.x diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 563701b8677..44a33eb7bcf 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -283,9 +283,6 @@ Introduced in GitLab 11.3. This file lives in `/var/log/gitlab/gitlab-rails/impo Omnibus GitLab packages or in `/home/git/gitlab/log/importer.log` for installations from source. -Currently it logs the progress of project imports from the Bitbucket Server -importer. Future importers may use this file. - ## `auth.log` Introduced in GitLab 12.0. This file lives in `/var/log/gitlab/gitlab-rails/auth.log` for diff --git a/doc/administration/monitoring/performance/img/performance_bar.png b/doc/administration/monitoring/performance/img/performance_bar.png Binary files differindex 2bf686f9017..8a6f8b3b273 100644 --- a/doc/administration/monitoring/performance/img/performance_bar.png +++ b/doc/administration/monitoring/performance/img/performance_bar.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png b/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png Binary files differindex 7af6d401d1d..265178729c4 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png +++ b/doc/administration/monitoring/performance/img/performance_bar_gitaly_calls.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png b/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png Binary files differnew file mode 100644 index 00000000000..78dd7594adf --- /dev/null +++ b/doc/administration/monitoring/performance/img/performance_bar_redis_calls.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png b/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png Binary files differnew file mode 100644 index 00000000000..f4068268137 --- /dev/null +++ b/doc/administration/monitoring/performance/img/performance_bar_rugged_calls.png diff --git a/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png b/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png Binary files differindex b3219b4fa94..dab323eb066 100644 --- a/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png +++ b/doc/administration/monitoring/performance/img/performance_bar_sql_queries.png diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index 4ee156fdc11..15eab7dcab0 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -8,15 +8,16 @@ activated, it looks as follows: It allows you to see (from left to right): - the current host serving the page -- the timing of the page (backend, frontend) - time taken and number of DB queries, click through for details of these queries  - time taken and number of [Gitaly] calls, click through for details of these calls  +- time taken and number of [Rugged] calls, click through for details of these calls +  - profile of the code used to generate the page, line by line. In the profile view, the numbers in the left panel represent wall time, cpu time, and number of calls (based on [rblineprof](https://github.com/tmm1/rblineprof)).  -- time taken and number of calls to Redis -- time taken and number of background jobs created by Sidekiq +- time taken and number of Redis calls, click through for details of these calls +  - time taken and number of Ruby GC calls On the far right is a request selector that allows you to view the same metrics @@ -43,3 +44,4 @@ You can toggle the Bar using the same shortcut.  [Gitaly]: ../../gitaly/index.md +[Rugged]: ../../high_availability/nfs.md#improving-nfs-performance-with-gitlab diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 098d946a9fa..604dff5983d 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -209,7 +209,7 @@ ps auwx | grep unicorn | awk '{ print " -p " $2}' | xargs strace -tt -T -f -s 10 The output in `/tmp/unicorn.txt` may help diagnose the root cause. -# More information +## More information - [Debugging Stuck Ruby Processes](https://blog.newrelic.com/2013/04/29/debugging-stuck-ruby-processes-what-to-do-before-you-kill-9/) - [Cheatsheet of using gdb and ruby processes](gdb-stuck-ruby.txt) diff --git a/doc/api/README.md b/doc/api/README.md index f9cc1a09870..91bc8ca8aab 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -29,7 +29,7 @@ The following API resources are available in the project context: | [Commits](commits.md) | `/projects/:id/repository/commits`, `/projects/:id/statuses` | | [Container Registry](container_registry.md) | `/projects/:id/registry/repositories` | | [Custom attributes](custom_attributes.md) | `/projects/:id/custom_attributes` (also available for groups and users) | -| [Dependencies](dependencies.md) **[ULTIMATE]** | `/projects/:id/dependencies` +| [Dependencies](dependencies.md) **(ULTIMATE)** | `/projects/:id/dependencies` | [Deploy keys](deploy_keys.md) | `/projects/:id/deploy_keys` (also available standalone) | | [Deployments](deployments.md) | `/projects/:id/deployments` | | [Discussions](discussions.md) (threaded comments) | `/projects/:id/issues/.../discussions`, `/projects/:id/snippets/.../discussions`, `/projects/:id/merge_requests/.../discussions`, `/projects/:id/commits/.../discussions` (also available for groups) | diff --git a/doc/api/epics.md b/doc/api/epics.md index d05eb0a8804..3036b3c2364 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -10,7 +10,7 @@ If epics feature is not available a `403` status code will be returned. The [epic issues API](epic_issues.md) allows you to interact with issues associated with an epic. -# Milestone dates integration +## Milestone dates integration > [Introduced][ee-6448] in GitLab 11.3. diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index efdcaf5a6f5..9f1ce1fc230 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -6,7 +6,6 @@ type: concepts, howto GitLab CI/CD allows you to use Docker Engine to build and test docker-based projects. - One of the new trends in Continuous Integration/Deployment is to: 1. Create an application image. @@ -29,7 +28,16 @@ during jobs. ## Runner Configuration -There are three methods to enable the use of `docker build` and `docker run` during jobs; each with their own tradeoffs. +There are three methods to enable the use of `docker build` and `docker run` +during jobs; each with their own tradeoffs. + +An alternative to using `docker build` is to [use kaniko](using_kaniko.md). +This avoids having to execute Runner in privileged mode. + +TIP: **Tip:** +To see how Docker and Runner are configured for shared Runners on +GitLab.com, see [GitLab.com Shared +Runners](../../user/gitlab_com/index.md#shared-runners). ### Use shell executor @@ -88,9 +96,9 @@ For more information please read [On Docker security: `docker` group considered The second approach is to use the special docker-in-docker (dind) [Docker image](https://hub.docker.com/_/docker/) with all tools installed (`docker`) and run the job script in context of that -image in privileged mode. +image in privileged mode. -NOTE: **Note:** `docker-compose` is not part of docker-in-docker (dind). In case you'd like to use `docker-compose` in your CI builds, please follow the [installation instructions for docker-compose](https://docs.docker.com/compose/install/) provided by docker. +NOTE: **Note:** `docker-compose` is not part of docker-in-docker (dind). In case you'd like to use `docker-compose` in your CI builds, please follow the [installation instructions for docker-compose](https://docs.docker.com/compose/install/) provided by docker. In order to do that, follow the steps: @@ -115,6 +123,13 @@ In order to do that, follow the steps: want to use [docker-in-docker] mode, you always have to use `privileged = true` in your Docker containers. + DANGER: **Danger:** + By enabling `--docker-privileged`, you are effectively disabling all of + the security mechanisms of containers and exposing your host to privilege + escalation which can lead to container breakout. For more information, check + out the official Docker documentation on + [Runtime privilege and Linux capabilities][docker-cap]. + The above command will create a `config.toml` entry similar to this: ```toml @@ -173,11 +188,6 @@ In order to do that, follow the steps: Docker-in-Docker works well, and is the recommended configuration, but it is not without its own challenges: -- By enabling `--docker-privileged`, you are effectively disabling all of - the security mechanisms of containers and exposing your host to privilege - escalation which can lead to container breakout. For more information, check - out the official Docker documentation on - [Runtime privilege and Linux capabilities][docker-cap]. - When using docker-in-docker, each job is in a clean environment without the past history. Concurrent jobs work fine because every build gets it's own instance of Docker engine so they won't conflict with each other. But this diff --git a/doc/ci/pipelines.md b/doc/ci/pipelines.md index 06a81c3d0c7..98f30350968 100644 --- a/doc/ci/pipelines.md +++ b/doc/ci/pipelines.md @@ -345,7 +345,7 @@ GitLab provides API endpoints to: - [Triggering pipelines through the API](triggers/README.md). - [Pipeline triggers API](../api/pipeline_triggers.md). -### Start multiple manual actions in a stage +### Start multiple manual actions in a stage > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/27188) in GitLab 11.11. diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index eb6dde2d24e..961520db7d8 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -69,7 +69,7 @@ curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10&token=se This endpoint attempts to fully utilise a single core, at 100%, for the given period. -Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). +Depending on your rack server setup, your request may timeout after a predetermined period (normally 60 seconds). If you're using Unicorn, this is done by killing the worker process. ``` @@ -80,7 +80,7 @@ GET /-/chaos/cpu_spin?duration_s=50&async=true | Attribute | Type | Required | Description | | ------------ | ------- | -------- | --------------------------------------------------------------------- | -| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilized. Defaults to 30s | | `async` | boolean | no | Set to true to consume CPU in a Sidekiq background worker process | ```bash @@ -93,7 +93,7 @@ curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60&token=secret This endpoint attempts to fully utilise a single core, and interleave it with DB request, for the given period. This endpoint can be used to model yielding execution to another threads when running concurrently. -Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). +Depending on your rack server setup, your request may timeout after a predetermined period (normally 60 seconds). If you're using Unicorn, this is done by killing the worker process. ``` @@ -105,7 +105,7 @@ GET /-/chaos/db_spin?duration_s=50&async=true | Attribute | Type | Required | Description | | ------------ | ------- | -------- | --------------------------------------------------------------------------- | | `interval_s` | float | no | Interval, in seconds, for every DB request. Defaults to 1s | -| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilized. Defaults to 30s | | `async` | boolean | no | Set to true to perform the operation in a Sidekiq background worker process | ```bash diff --git a/doc/development/contributing/community_roles.md b/doc/development/contributing/community_roles.md index 3296cb173d7..7d2d1b77a0e 100644 --- a/doc/development/contributing/community_roles.md +++ b/doc/development/contributing/community_roles.md @@ -1,4 +1,4 @@ -### Community members & roles +# Community members & roles GitLab community members and their privileges/responsibilities. diff --git a/doc/development/fe_guide/axios.md b/doc/development/fe_guide/axios.md index 0d9397c3bd5..09b4a3c3d96 100644 --- a/doc/development/fe_guide/axios.md +++ b/doc/development/fe_guide/axios.md @@ -1,15 +1,18 @@ # Axios + We use [axios][axios] to communicate with the server in Vue applications and most new code. In order to guarantee all defaults are set you *should not use `axios` directly*, you should import `axios` from `axios_utils`. ## CSRF token + All our request require a CSRF token. To guarantee this token is set, we are importing [axios][axios], setting the token, and exporting `axios` . This exported module should be used instead of directly using `axios` to ensure the token is set. ## Usage + ```javascript import axios from './lib/utils/axios_utils'; diff --git a/doc/development/fe_guide/style_guide_scss.md b/doc/development/fe_guide/style_guide_scss.md index f895cfd36ac..95c4a094c04 100644 --- a/doc/development/fe_guide/style_guide_scss.md +++ b/doc/development/fe_guide/style_guide_scss.md @@ -212,6 +212,7 @@ selectors are intended for use only with JavaScript to allow for removal or renaming without breaking styling. ### IDs + Don't use ID selectors in CSS. ```scss diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index f827d240bf6..9f0ac8cc753 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -173,6 +173,45 @@ func Test(t *testing.T) { } ``` +### Table-Driven Tests + +Using [Table-Driven Tests](https://github.com/golang/go/wiki/TableDrivenTests) +is generally good practice when you have multiple entries of +inputs/outputs for the same function. Below are some guidelines one can +follow when writing table-driven test. These guidelines are mostly +extracted from Go standard library source code. Keep in mind it's OK not +to follow these guidelines when it makes sense. + +#### Defining test cases + +Each table entry is a complete test case with inputs and expected +results, and sometimes with additional information such as a test name +to make the test output easily readable. + +- [Define a slice of anonymous struct](https://github.com/golang/go/blob/50bd1c4d4eb4fac8ddeb5f063c099daccfb71b26/src/encoding/csv/reader_test.go#L16) + inside of the test. +- [Define a slice of anonymous struct](https://github.com/golang/go/blob/55d31e16c12c38d36811bdee65ac1f7772148250/src/cmd/go/internal/module/module_test.go#L9-L66) + outside of the test. +- [Named structs](https://github.com/golang/go/blob/2e0cd2aef5924e48e1ceb74e3d52e76c56dd34cc/src/cmd/go/internal/modfetch/coderepo_test.go#L54-L69) + for code reuse. +- [Using `map[string]struct{}`](https://github.com/golang/go/blob/6d5caf38e37bf9aeba3291f1f0b0081f934b1187/src/cmd/trace/annotations_test.go#L180-L235). + +#### Contents of the test case + +- Ideally, each test case should have a field with a unique identifier + to use for naming subtests. In the Go standard library, this is commonly the + `name string` field. +- Use `want`/`expect`/`actual` when you are specifcing something in the + test case that will be used for assertion. + +#### Variable names + +- Each table-driven test map/slice of struct can be named `tests`. +- When looping through `tests` the anonymous struct can be referred + to as `tt` or `tc`. +- The description of the test can be referred to as + `name`/`testName`/`tn`. + ### Benchmarks Programs handling a lot of IO or complex operations should always include diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 4c78c62f5f5..0c343bc22e4 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -19,6 +19,7 @@ Project.find_by_full_path('group/project').import_state.slice(:jid, :status, :la grep JID /var/log/gitlab/sidekiq/current grep "Import/Export error" /var/log/gitlab/sidekiq/current grep "Import/Export backtrace" /var/log/gitlab/sidekiq/current +tail /var/log/gitlab/gitlab-rails/importer.log ``` ## Troubleshooting performance issues diff --git a/doc/development/module_with_instance_variables.md b/doc/development/module_with_instance_variables.md index 7bdfa04fc57..443eee0b62c 100644 --- a/doc/development/module_with_instance_variables.md +++ b/doc/development/module_with_instance_variables.md @@ -1,6 +1,6 @@ -## Modules with instance variables could be considered harmful +# Modules with instance variables could be considered harmful -### Background +## Background Rails somehow encourages people using modules and instance variables everywhere. For example, using instance variables in the controllers, @@ -9,7 +9,7 @@ helpers, and views. They're also encouraging the use of saving everything in a giant, single object, and people could access everything in that one giant object. -### The problems +## The problems Of course this is convenient to develop, because we just have everything within reach. However this has a number of downsides when that chosen object @@ -24,7 +24,7 @@ manipulated from 3 different modules. It's hard to track when those variables start giving us troubles. We don't know which module would suddenly change one of the variables. Everything could touch anything. -### Similar concerns +## Similar concerns People are saying multiple inheritance is bad. Mixing multiple modules with multiple instance variables scattering everywhere suffer from the same issue. @@ -40,7 +40,7 @@ Note that `included` doesn't solve the whole issue. They define the dependencies, but they still allow each modules to talk implicitly via the instance variables in the final giant object, and that's where the problem is. -### Solutions +## Solutions We should split the giant object into multiple objects, and they communicate with each other with the API, i.e. public methods. In short, composition over @@ -53,7 +53,7 @@ With clearly defined API, this would make things less coupled and much easier to debug and track, and much more extensible for other objects to use, because they communicate in a clear way, rather than implicit dependencies. -### Acceptable use +## Acceptable use However, it's not always bad to use instance variables in a module, as long as it's contained in the same module; that is, no other modules or @@ -74,7 +74,7 @@ Unfortunately it's not easy to code more complex rules into the cop, so we rely on people's best judgement. If we could find another good pattern we could easily add to the cop, we should do it. -### How to rewrite and avoid disabling this cop +## How to rewrite and avoid disabling this cop Even if we could just disable the cop, we should avoid doing so. Some code could be easily rewritten in simple form. Consider this acceptable method: @@ -181,7 +181,7 @@ rather than whatever includes the module, and those modules which were also included, making it much easier to track down any issues, and reducing the chance of having name conflicts. -### How to disable this cop +## How to disable this cop Put the disabling comment right after your code in the same line: @@ -210,14 +210,14 @@ end Note that you need to enable it at some point, otherwise everything below won't be checked. -### Things we might need to ignore right now +## Things we might need to ignore right now Because of the way Rails helpers and mailers work, we might not be able to avoid the use of instance variables there. For those cases, we could ignore them at the moment. At least we're not going to share those modules with other random objects, so they're still somewhat isolated. -### Instance variables in views +## Instance variables in views They're bad because we can't easily tell who's using the instance variables (from controller's point of view) and where we set them up (from partials' diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index 81a29170129..ae5c4c6a6cc 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -1,17 +1,21 @@ # Accessiblity + Using semantic HTML plays a key role when it comes to accessibility. ## Accessible Rich Internet Applications - ARIA + WAI-ARIA, the Accessible Rich Internet Applications specification, defines a way to make Web content and Web applications more accessible to people with disabilities. > Note: It is [recommended][using-aria] to use semantic elements as the primary method to achieve accessibility rather than adding aria attributes. Adding aria attributes should be seen as a secondary method for creating accessible elements. ### Role + The `role` attribute describes the role the element plays in the context of the document. Check the list of WAI-ARIA roles [here][roles] ## Icons + When using icons or images that aren't absolutely needed to understand the context, we should use `aria-hidden="true"`. On the other hand, if an icon is crucial to understand the context we should do one of the following: @@ -20,6 +24,7 @@ On the other hand, if an icon is crucial to understand the context we should do 1. Use `aria-labelledby` to point to an element that contains the explanation for that icon ## Form inputs + In forms we should use the `for` attribute in the label statement: ``` diff --git a/doc/development/new_fe_guide/style/index.md b/doc/development/new_fe_guide/style/index.md index 335d9e66240..f073dc56f1f 100644 --- a/doc/development/new_fe_guide/style/index.md +++ b/doc/development/new_fe_guide/style/index.md @@ -8,7 +8,7 @@ ## [Vue style guide](vue.md) -# Tooling +## Tooling ## [Prettier](prettier.md) diff --git a/doc/development/reusing_abstractions.md b/doc/development/reusing_abstractions.md index 59da02ed6fd..fce144f8dc2 100644 --- a/doc/development/reusing_abstractions.md +++ b/doc/development/reusing_abstractions.md @@ -14,13 +14,13 @@ on maintainability, the ability to easily debug problems, or even performance. An example would be to use `ProjectsFinder` in `IssuesFinder` to limit issues to those belonging to a set of projects. While initially this may seem like a good idea, both classes provide a very high level interface with very little control. -This means that `IssuesFinder` may not be able to produce a better optimised +This means that `IssuesFinder` may not be able to produce a better optimized database query, as a large portion of the query is controlled by the internals of `ProjectsFinder`. To work around this problem, you would use the same code used by `ProjectsFinder`, instead of using `ProjectsFinder` itself directly. This allows -you to compose your behaviour better, giving you more control over the behaviour +you to compose your behavior better, giving you more control over the behavior of the code. To illustrate, consider the following code from `IssuableFinder#projects`: @@ -52,7 +52,7 @@ functionality is added to this (high level) interface. Instead of _only_ affecting the cases where this is necessary, it may also start affecting `IssuableFinder` in a negative way. For example, the query produced by `GroupProjectsFinder` may include unnecessary conditions. Since we're using a -finder here, we can't easily opt-out of that behaviour. We could add options to +finder here, we can't easily opt-out of that behavior. We could add options to do so, but then we'd need as many options as we have features. Every option adds two code paths, which means that for four features we have to cover 8 different code paths. @@ -213,6 +213,5 @@ The API provided by Active Record itself, such as the `where` method, `save`, Everything in `app/workers`. -The scheduling of Sidekiq jobs using `SomeWorker.perform_async`, `perform_in`, -etc. Directly invoking a worker using `SomeWorker.new.perform` should be avoided -at all times in application code, though this is fine to use in tests. +Use `SomeWorker.perform_async` or `SomeWorker.perform_in` to schedule Sidekiq +jobs. Never directly invoke a worker using `SomeWorker.new.perform`. diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index 661ab9cef1a..ccec6f7d719 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -25,7 +25,7 @@ else end ``` -# Read-only database +## Read-only database The database can be used in read-only mode. In this case we have to make sure all GET requests don't attempt any write operations to the diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 30b9e88ade8..c416dbac492 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -1,6 +1,7 @@ # Auto DevOps -> [Introduced][ce-37115] in GitLab 10.0. Generally available on GitLab 11.0. +> - [Introduced][ce-37115] in GitLab 10.0. +> - Generally available on GitLab 11.0. Auto DevOps provides pre-defined CI/CD configuration which allows you to automatically detect, build, test, deploy, and monitor your applications. Leveraging CI/CD best practices and tools, Auto DevOps aims @@ -43,16 +44,16 @@ platform or a Platform as a Service (PaaS). It takes inspiration from the innovative work done by [Heroku](https://www.heroku.com/) and goes beyond it in multiple ways: -1. Auto DevOps works with any Kubernetes cluster; you're not limited to running - on GitLab's infrastructure. (Note that many features also work without Kubernetes.) -1. There is no additional cost (no markup on the infrastructure costs), and you - can use a self-hosted Kubernetes cluster or Containers as a Service on any - public cloud (for example, [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/)). -1. Auto DevOps has more features including security testing, performance testing, - and code quality testing. -1. Auto DevOps offers an incremental graduation path. If you need advanced customizations, - you can start modifying the templates without having to start over on a - completely different platform. Review the [customizing](#customizing) section for more information. +- Auto DevOps works with any Kubernetes cluster; you're not limited to running + on GitLab's infrastructure. (Note that many features also work without Kubernetes). +- There is no additional cost (no markup on the infrastructure costs), and you + can use a self-hosted Kubernetes cluster or Containers as a Service on any + public cloud (for example, [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/)). +- Auto DevOps has more features including security testing, performance testing, + and code quality testing. +- Auto DevOps offers an incremental graduation path. If you need advanced customizations, + you can start modifying the templates without having to start over on a + completely different platform. Review the [customizing](#customizing) section for more information. ## Features @@ -90,38 +91,38 @@ For an overview on the creation of Auto DevOps, read the blog post [From 2/3 of To make full use of Auto DevOps, you will need: -1. **GitLab Runner** (needed for all stages) - Your Runner needs to be - configured to be able to run Docker. Generally this means using the - [Docker](https://docs.gitlab.com/runner/executors/docker.html) or [Kubernetes - executor](https://docs.gitlab.com/runner/executors/kubernetes.html), with - [privileged mode enabled](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). - The Runners do not need to be installed in the Kubernetes cluster, but the - Kubernetes executor is easy to use and is automatically autoscaling. - Docker-based Runners can be configured to autoscale as well, using [Docker - Machine](https://docs.gitlab.com/runner/install/autoscaling.html). Runners - should be registered as [shared Runners](../../ci/runners/README.md#registering-a-shared-runner) - for the entire GitLab instance, or [specific Runners](../../ci/runners/README.md#registering-a-specific-runner) - that are assigned to specific projects. -1. **Base domain** (needed for Auto Review Apps and Auto Deploy) - You will need - a domain configured with wildcard DNS which is going to be used by all of your - Auto DevOps applications. [Read the specifics](#auto-devops-base-domain). -1. **Kubernetes** (needed for Auto Review Apps, Auto Deploy, and Auto Monitoring) - - To enable deployments, you will need Kubernetes 1.5+. You need a [Kubernetes cluster][kubernetes-clusters] - for the project, or a Kubernetes [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation. - 1. **A load balancer** - You can use NGINX ingress by deploying it to your - Kubernetes cluster using the - [`nginx-ingress`](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress) - Helm chart. -1. **Prometheus** (needed for Auto Monitoring) - To enable Auto Monitoring, you - will need Prometheus installed somewhere (inside or outside your cluster) and - configured to scrape your Kubernetes cluster. To get response metrics - (in addition to system metrics), you need to - [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). - The [Prometheus service](../../user/project/integrations/prometheus.md) - integration needs to be enabled for the project, or enabled as a - [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation. +- **GitLab Runner** (needed for all stages) - Your Runner needs to be + configured to be able to run Docker. Generally this means using the + [Docker](https://docs.gitlab.com/runner/executors/docker.html) or [Kubernetes + executor](https://docs.gitlab.com/runner/executors/kubernetes.html), with + [privileged mode enabled](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). + The Runners do not need to be installed in the Kubernetes cluster, but the + Kubernetes executor is easy to use and is automatically autoscaling. + Docker-based Runners can be configured to autoscale as well, using [Docker + Machine](https://docs.gitlab.com/runner/install/autoscaling.html). Runners + should be registered as [shared Runners](../../ci/runners/README.md#registering-a-shared-runner) + for the entire GitLab instance, or [specific Runners](../../ci/runners/README.md#registering-a-specific-runner) + that are assigned to specific projects. +- **Base domain** (needed for Auto Review Apps and Auto Deploy) - You will need + a domain configured with wildcard DNS which is going to be used by all of your + Auto DevOps applications. [Read the specifics](#auto-devops-base-domain). +- **Kubernetes** (needed for Auto Review Apps, Auto Deploy, and Auto Monitoring) - + To enable deployments, you will need Kubernetes 1.5+. You need a [Kubernetes cluster][kubernetes-clusters] + for the project, or a Kubernetes [default service template](../../user/project/integrations/services_templates.md) + for the entire GitLab installation. + - **A load balancer** - You can use NGINX ingress by deploying it to your + Kubernetes cluster using the + [`nginx-ingress`](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress) + Helm chart. +- **Prometheus** (needed for Auto Monitoring) - To enable Auto Monitoring, you + will need Prometheus installed somewhere (inside or outside your cluster) and + configured to scrape your Kubernetes cluster. To get response metrics + (in addition to system metrics), you need to + [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). + The [Prometheus service](../../user/project/integrations/prometheus.md) + integration needs to be enabled for the project, or enabled as a + [default service template](../../user/project/integrations/services_templates.md) + for the entire GitLab installation. If you do not have Kubernetes or Prometheus installed, then Auto Review Apps, Auto Deploy, and Auto Monitoring will be silently skipped. @@ -147,7 +148,7 @@ NOTE: **Note** A wildcard DNS A record matching the base domain(s) is required, for example, given a base domain of `example.com`, you'd need a DNS entry like: -``` +```text *.example.com 3600 A 1.2.3.4 ``` @@ -224,7 +225,7 @@ full use of Auto DevOps are available. If this is your fist time, we recommend y GitLab.com users can enable/disable Auto DevOps at the project-level only. Self-managed users can enable/disable Auto DevOps at the project-level, group-level or instance-level. -### Enabling/disabling Auto DevOps at the instance-level (Administrators only) +### At the instance level (Administrators only) Even when disabled at the instance level, group owners and project maintainers can still enable Auto DevOps at the group and project level, respectively. @@ -234,7 +235,7 @@ Auto DevOps at the group and project level, respectively. 1. If enabling, optionally set up the Auto DevOps [base domain](#auto-devops-base-domain) which will be used for Auto Deploy and Auto Review Apps. 1. Click **Save changes** for the changes to take effect. -### Enabling/disabling Auto DevOps at the group-level +### At the group level > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/52447) in GitLab 11.10. @@ -250,7 +251,7 @@ When enabling or disabling Auto DevOps at group-level, group configuration will the subgroups and projects inside that group, unless Auto DevOps is specifically enabled or disabled on the subgroup or project. -### Enabling/disabling Auto DevOps at the project-level +### At the project level If enabling, check that your project doesn't have a `.gitlab-ci.yml`, or if one exists, remove it. @@ -263,7 +264,7 @@ If enabling, check that your project doesn't have a `.gitlab-ci.yml`, or if one When the feature has been enabled, an Auto DevOps pipeline is triggered on the default branch. -### Feature flag to enable for a percentage of projects +### Enable for a percentage of projects There is also a feature flag to enable Auto DevOps by default to your chosen percentage of projects. @@ -371,7 +372,7 @@ Any differences between the source and target branches are also Static Application Security Testing (SAST) uses the [SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static analysis on the current code and checks for potential security issues. The -the Auto SAST stage will be skipped on licenses other than Ultimate and requires GitLab Runner 11.5 or above. +Auto SAST stage will be skipped on licenses other than Ultimate and requires GitLab Runner 11.5 or above. Once the report is created, it's uploaded as an artifact which you can later download and check out. @@ -488,7 +489,7 @@ Any security warnings are also shown in the merge request widget. Read how Auto Browser Performance Testing utilizes the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) to measure the performance of a web page. A JSON report is created and uploaded as an artifact, which includes the overall performance score for each page. By default, the root page of Review and Production environments will be tested. If you would like to add additional URL's to test, simply add the paths to a file named `.gitlab-urls.txt` in the root directory, one per line. For example: -``` +```text / /features /direction @@ -749,7 +750,7 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. | | `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). | -| `INCREMENTAL_ROLLOUT_MODE`| From GitLab 11.4, this variable, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment.<br/>Set to: <ul><li>`manual`, for manual deployment jobs.</li><li>`timed`, for automatic rollout deployments with a 5 minute delay each one.</li></ul> | +| `INCREMENTAL_ROLLOUT_MODE`| From GitLab 11.4, this variable, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | | `TEST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. | | `CODE_QUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. | | `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, this variable can be used to disable the `license_management` job. If the variable is present, the job will not be created. | @@ -1009,7 +1010,7 @@ multi-buildpack does not. As of GitLab 10.0, the supported buildpacks are: -``` +```text - heroku-buildpack-multi v1.0.0 - heroku-buildpack-ruby v168 - heroku-buildpack-nodejs v99 diff --git a/doc/user/abuse_reports.md b/doc/user/abuse_reports.md index 41ee7e62b2c..e6c86cc8f2e 100644 --- a/doc/user/abuse_reports.md +++ b/doc/user/abuse_reports.md @@ -1,6 +1,12 @@ # Abuse reports -Report abuse from users to GitLab administrators. +You can report abuse from other GitLab users to GitLab administrators. + +A GitLab administrator [can then choose](admin_area/abuse_reports.md) to: + +- Remove the user, which deletes them from the instance. +- Block the user, which denies them access to the instance. +- Or remove the report, which retains the users access to the instance. You can report a user through their: @@ -12,7 +18,8 @@ You can report a user through their: To report abuse from a user's profile page: -1. Click on the exclamation point report abuse button at the top right of the user's profile. +1. Click on the exclamation point report abuse button at the top right of the + user's profile. 1. Complete an abuse report. 1. Click the **Send report** button. @@ -26,15 +33,18 @@ To report abuse from a user's comment: 1. Click the **Send report** button. NOTE: **Note:** -A URL to the reported user's comment will be -pre-filled in the abuse report's **Message** field. +A URL to the reported user's comment will be pre-filled in the abuse report's +**Message** field. ## Reporting abuse through a user's issue or merge request The **Report abuse** button is displayed at the top right of the issue or merge request: -- When **Report abuse** is selected from the menu that appears when the **Close issue** or **Close merge request** button is clicked, for users that have permission to close the issue or merge request. -- When viewing the issue or merge request, for users that don't have permission to close the issue or merge request. +- When **Report abuse** is selected from the menu that appears when the + **Close issue** or **Close merge request** button is clicked, for users that + have permission to close the issue or merge request. +- When viewing the issue or merge request, for users that don't have permission + to close the issue or merge request. With the **Report abuse** button displayed, to submit an abuse report: diff --git a/doc/user/admin_area/abuse_reports.md b/doc/user/admin_area/abuse_reports.md index 01c2d9607f5..8088c33fc9c 100644 --- a/doc/user/admin_area/abuse_reports.md +++ b/doc/user/admin_area/abuse_reports.md @@ -2,30 +2,60 @@ View and resolve abuse reports from GitLab users. -Admins can view abuse reports in the admin area and are able to -resolve the reports by removing the reported user, blocking the reported user, or removing the report. +GitLab administrators can view and [resolve](#resolving-abuse-reports) abuse +reports in the Admin Area. ## Reporting abuse -To find out more about reporting abuse, see [abuse reports user documentation](../abuse_reports.md). +To find out more about reporting abuse, see [abuse reports user +documentation](../abuse_reports.md). ## Resolving abuse reports -To access abuse reports, go to **Admin area > Abuse Reports**. +To access abuse reports, go to **Admin Area > Abuse Reports**. There are 3 ways to resolve an abuse report, with a button for each method: -- Remove user & report: [Deletes the reported user](../profile/account/delete_account.md) from the instance and removes the abuse report from the list. -- Block user: Blocks the reported user from the instance and does not remove the abuse report from the list. -- Remove report: Removes the abuse report from the list and does not restrict the access for the reported user. +- Remove user & report. This will: + - [Delete the reported user](../profile/account/delete_account.md) from the + instance. + - Remove the abuse report from the list. +- [Block user](#blocking-users). +- Remove report. This will: + - Remove the abuse report from the list. + - Remove access restrictions for the reported user. + +The following is an example of the **Abuse Reports** page:  -## Blocked users +### Blocking users + +A blocked user cannot log in or access any repositories, but all of their data +remains. + +Blocking a user: + +- Leaves them in the abuse report list. +- Changes the **Block user** button to a disabled **Already blocked** button. -Blocking a user will not remove the abuse report from the list. +The user will be notified with the +[following message](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/app/workers/email_receiver_worker.rb#L38): -Instead, the block button will be disabled and explain that the user is "Already blocked". -You are still able to remove the user and/or report if necessary. +```text +Your account has been blocked. If you believe this is in error, contact a staff member. +``` + +After blocking, you can still either: + +- Remove the user and report if necessary. +- Remove the report. + +The following is an example of a blocked user listed on the **Abuse Reports** +page:  + +NOTE: **Note:** +Users can be [blocked](../../api/users.md#block-user) and +[unblocked](../../api/users.md#unblock-user) using the GitLab API. diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index f698e0a1608..516600c9d99 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -52,8 +52,8 @@ You can view the exact JSON payload in the administration panel. To view the pay 1. Expand **Settings** in the left sidebar and click on **Metrics and profiling**. 1. Expand **Usage statistics** and click on the **Preview payload** button. -You can see how [the usage ping data maps to different stages of the product](https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv). - +You can see how [the usage ping data maps to different stages of the product](https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv). + ### Deactivate the usage ping The usage ping is opt-out. If you want to deactivate this feature, go to diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index 99589cb1915..625c5440ec0 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -137,6 +137,13 @@ The result will then be: - The Staging cluster will be used for the `deploy to staging` job. - The Production cluster will be used for the `deploy to production` job. +## Security of Runners + +For important information about securely configuring GitLab Runners, see +[Security of +Runners](../../project/clusters/index.md#security-of-gitlab-runners) +documentation for project-level clusters. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 044be05b932..e6822f0c52c 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -47,6 +47,7 @@ The following table depicts the various user permission levels in a project. | View approved/blacklisted licenses **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View license management reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| View [Design Management](project/issues/design_management.md) pages **(PREMIUM)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View project code | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Pull project code | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View GitLab Pages protected by [access control](project/pages/introduction.md#gitlab-pages-access-control-core-only) | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -74,6 +75,7 @@ The following table depicts the various user permission levels in a project. | View Error Tracking list | | ✓ | ✓ | ✓ | ✓ | | Pull from [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | Publish to [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ || +| Upload [Design Management](project/issues/design_management.md) files **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | | Push to non-protected branches | | | ✓ | ✓ | ✓ | | Force push to non-protected branches | | | ✓ | ✓ | ✓ | diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index 304a7984191..cbee79de493 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -1,37 +1,71 @@ -# Deleting a User Account +# Deleting a User account + +Users can be deleted from a GitLab instance, either by: + +- The user themselves. +- An administrator. NOTE: **Note:** Deleting a user will delete all projects in that user namespace. -- As a user, you can delete your own account by navigating to **Settings** > **Account** and selecting **Delete account** -- As an admin, you can delete a user account by navigating to the **Admin Area**, selecting the **Users** tab, selecting a user, and clicking on **Delete user** +## As a user + +As a user, you can delete your own account by: + +1. Clicking on your avatar. +1. Navigating to **Settings > Account**. +1. Selecting **Delete account**. + +## As an administrator + +As an administrator, you can delete a user account by: + +1. Navigating to **Admin Area > Overview > Users**. +1. Selecting a user. +1. Under the **Account** tab, clicking: + - **Delete user** to delete only the user but maintaining their + [associated records](#associated-records). + - **Delete user and contributions** to delete the user and + their associated records. + +### Blocking a user + +In addition to blocking a user +[via an abuse report](../../admin_area/abuse_reports.md#blocking-users), +a user can be blocked directly from the Admin area. To do this: + +1. Navigate to **Admin Area > Overview > Users**. +1. Selecting a user. +1. Under the **Account** tab, click **Block user**. ## Associated Records -> Introduced for issues in [GitLab 9.0][ce-7393], and for merge requests, award - emoji, notes, and abuse reports in [GitLab 9.1][ce-10467]. - Hard deletion from abuse reports and spam logs was introduced in - [GitLab 9.1][ce-10273], and from the API in [GitLab 9.3][ce-11853]. +> - Introduced for issues in +> [GitLab 9.0](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7393). +> - Introduced for merge requests, award emoji, notes, and abuse reports in +> [GitLab 9.1](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10467). +> - Hard deletion from abuse reports and spam logs was introduced in +> [GitLab 9.1](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10273), +> and from the API in +> [GitLab 9.3](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11853). When a user account is deleted, not all associated records are deleted with it. Here's a list of things that will **not** be deleted: -- Issues that the user created -- Merge requests that the user created -- Notes that the user created -- Abuse reports that the user reported -- Award emoji that the user created +- Issues that the user created. +- Merge requests that the user created. +- Notes that the user created. +- Abuse reports that the user reported. +- Award emoji that the user created. Instead of being deleted, these records will be moved to a system-wide -user with the username "Ghost User", whose sole purpose is to act as a container for such records. Any commits made by a deleted user will still display the username of the original user. - -When a user is deleted from an [abuse report](../../admin_area/abuse_reports.md) or spam log, these associated -records are not ghosted and will be removed, along with any groups the user -is a sole owner of. Administrators can also request this behaviour when -deleting users from the [API](../../../api/users.md#user-deletion) or the -admin area. - -[ce-7393]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7393 -[ce-10273]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10273 -[ce-10467]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10467 -[ce-11853]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11853 +user with the username "Ghost User", whose sole purpose is to act as a container +for such records. Any commits made by a deleted user will still display the +username of the original user. + +When a user is deleted from an [abuse report](../../admin_area/abuse_reports.md) +or spam log, these associated records are not ghosted and will be removed, along +with any groups the user is a sole owner of. + +Administrators can also request this behavior when deleting users from the +[API](../../../api/users.md#user-deletion) or the Admin Area. diff --git a/doc/user/project/clusters/eks_and_gitlab/index.md b/doc/user/project/clusters/eks_and_gitlab/index.md index 55a9fbabf98..28f3420de35 100644 --- a/doc/user/project/clusters/eks_and_gitlab/index.md +++ b/doc/user/project/clusters/eks_and_gitlab/index.md @@ -253,7 +253,7 @@ With RBAC disabled and services deployed, [Auto DevOps](../../../../topics/autodevops/index.md) can now be leveraged to build, test, and deploy the app. -[Enable Auto DevOps](../../../../topics/autodevops/index.md#enablingdisabling-auto-devops-at-the-project-level) +[Enable Auto DevOps](../../../../topics/autodevops/index.md#at-the-project-level) if not already enabled. If a wildcard DNS entry was created resolving to the Load Balancer, enter it in the `domain` field under the Auto DevOps settings. Otherwise, the deployed app will not be externally available outside of the cluster. diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 4c247691757..35f2976899b 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -343,10 +343,15 @@ turn can do almost everything that the host can do. Be aware of the inherent security risk associated with performing `docker run` operations on arbitrary images as they effectively have root access. -If you don't want to use GitLab Runner in privileged mode, first make sure that -you don't have it installed via the applications, and then use the -[Runner's Helm chart](../../../install/kubernetes/gitlab_runner_chart.md) to -install it manually. +If you don't want to use GitLab Runner in privileged mode, either: + +- Use shared Runners on GitLab.com. They don't have this security issue. +- Set up your own Runners using configuration described at + [Shared Runners](../../gitlab_com/index.md#shared-runners). This involves: + 1. Making sure that you don't have it installed via + [the applications](#installing-applications). + 1. Installing a Runner + [using `docker+machine`](https://docs.gitlab.com/runner/executors/docker_machine.html). ## Installing applications diff --git a/doc/user/project/integrations/prometheus_library/haproxy.md b/doc/user/project/integrations/prometheus_library/haproxy.md index 6be8fc82431..90a725f271b 100644 --- a/doc/user/project/integrations/prometheus_library/haproxy.md +++ b/doc/user/project/integrations/prometheus_library/haproxy.md @@ -1,4 +1,5 @@ # Monitoring HAProxy + > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12621) in GitLab 9.4 GitLab has support for automatically detecting and monitoring HAProxy. This is provided by leveraging the [HAProxy Exporter](https://github.com/prometheus/haproxy_exporter), which translates HAProxy statistics into a Prometheus readable form. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md new file mode 100644 index 00000000000..2327fa84998 --- /dev/null +++ b/doc/user/project/issues/design_management.md @@ -0,0 +1,58 @@ +# Design Management **(PREMIUM)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. + +CAUTION: **Warning:** +This an __alpha__ feature and is subject to change at any time without +prior notice. + +## Overview + +Design Management allows you to upload design assets (wireframes, mockups, etc.) +to GitLab issues and keep them stored in one single place, accessed by the Design +Management's page within an issue, giving product designers, product managers, and engineers a +way to collaborate on designs over one single source of truth. + +You can easily share mock-ups of designs with your team, or visual regressions can be easily +viewed and addressed. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see the video [Design Management (GitLab 12.2)](https://www.youtube.com/watch?v=CCMtCqdK_aM). + +## Requirements + +Design Management requires +[Large File Storage (LFS)](../../../workflow/lfs/manage_large_binaries_with_git_lfs.md) +to be enabled: + +- For GitLab.com, LFS is already enabled. +- For self-managed instances, a GitLab administrator must have + [enabled LFS globally](../../../workflow/lfs/lfs_administration.md). +- For both GitLab.com and self-managed instances: LFS must be enabled for the project itself. + If enabled globally, LFS will be enabled by default to all projects. To enable LFS on the + project level, navigate to your project's **Settings > General**, expand **Visibility, project features, permissions** + and enable **Git Large File Storage**. + +## Limitations + +- Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, `gif`, `bmp`, `tiff` or `ico`. The [`svg` extension is not yet supported](https://gitlab.com/gitlab-org/gitlab-ee/issues/12771). +- [Designs cannot yet be deleted](https://gitlab.com/gitlab-org/gitlab-ee/issues/11089). +- Design Management is [not yet supported in the project export](https://gitlab.com/gitlab-org/gitlab-ee/issues/11090). + +## The Design Management page + +Navigate to the **Design Management** page from any issue by clicking the **Designs** tab: + + + +## Adding designs + +To upload design images, click the **Upload Designs** button and select images to upload. + +Designs with the same filename as an existing uploaded design will create a new version +of the design, and will replace the previous version. + +## Viewing designs + +Images on the Design Management page can be enlarged by clicking on them. + diff --git a/doc/user/project/issues/img/design_management_v12_2.png b/doc/user/project/issues/img/design_management_v12_2.png Binary files differnew file mode 100644 index 00000000000..6da747a3f21 --- /dev/null +++ b/doc/user/project/issues/img/design_management_v12_2.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 6706e1c2e2b..bf04ed2d2d0 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -120,6 +120,12 @@ associated label or assignee will change to match that of the new column. The en board can also be filtered to only include issues from a certain milestone or an overarching label. +### Design Management **(PREMIUM)** + +With [Design Management](design_management.md), you can upload design +assets to issues and view them all together to easily share and +collaborate with your team. + ### Epics **(ULTIMATE)** [Epics](../../group/epics/index.md) let you manage your portfolio of projects more diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index ad1d79ae5b1..eb6e454062a 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -9,8 +9,18 @@ in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. With the help of [GitLab CI/CD](../../../ci/README.md), you can analyze your source code quality using GitLab Code Quality. -Code Quality uses [Code Climate Engines](https://codeclimate.com), which are -free and open source. Code Quality doesn't require a Code Climate subscription. + +Code Quality: + +- Uses [Code Climate Engines](https://codeclimate.com), which are + free and open source. Code Quality doesn't require a Code Climate + subscription. +- Runs in [pipelines](../../../ci/pipelines.md) using an Docker image built in + [GitLab Code + Quality](https://gitlab.com/gitlab-org/security-products/codequality) project. +- Can make use of a [template](#template-and-examples). +- Is available with [Auto + DevOps](../../../topics/autodevops/index.md#auto-code-quality-starter). Going a step further, GitLab can show the Code Quality report right in the merge request widget area: @@ -21,22 +31,48 @@ in the merge request widget area: For instance, consider the following workflow: -1. Your backend team member starts a new implementation for making a certain feature in your app faster -1. With Code Quality reports, they analyze how their implementation is impacting the code quality -1. The metrics show that their code degrade the quality in 10 points -1. You ask a co-worker to help them with this modification -1. They both work on the changes until Code Quality report displays no degradations, only improvements -1. You approve the merge request and authorize its deployment to staging -1. Once verified, their changes are deployed to production +1. Your backend team member starts a new implementation for making a certain + feature in your app faster. +1. With Code Quality reports, they analyze how their implementation is impacting + the code quality. +1. The metrics show that their code degrade the quality in 10 points. +1. You ask a co-worker to help them with this modification. +1. They both work on the changes until Code Quality report displays no + degradations, only improvements. +1. You approve the merge request and authorize its deployment to staging. +1. Once verified, their changes are deployed to production. + +## Template and examples + +For most GitLab instances, the supplied template is the preferred method of +implementing Code Quality. See +[Analyze your project's Code Quality](../../../ci/examples/code_quality.md) for: + +- Information on the builtin GitLab Code Quality template. +- Examples of manual GitLab configuration for earlier GitLab versions. -## How it works +## Configuring jobs using variables -First of all, you need to define a job in your `.gitlab-ci.yml` file that generates the -[Code Quality report artifact](../../../ci/yaml/README.md#artifactsreportscodequality-starter). +The Code Quality job supports environment variables that users can set to +configure job execution at runtime. -The Code Quality report artifact is a subset of the -[Code Climate spec](https://github.com/codeclimate/spec/blob/master/SPEC.md#data-types). -It must be a JSON file containing an array of objects with the following properties: +For a list of available environment variables, see +[Environment variables](https://gitlab.com/gitlab-org/security-products/codequality/blob/master/README.md#environment-variables). + +## Implementing a custom tool + +It's possible to have a custom tool provide Code Quality reports in GitLab. To +do this: + +1. Define a job in your `.gitlab-ci.yml` file that generates the + [Code Quality report + artifact](../../../ci/yaml/README.md#artifactsreportscodequality-starter). +1. Configure your tool to generate the Code Quality report artifact as a JSON + file that implements subset of the [Code Climate + spec](https://github.com/codeclimate/spec/blob/master/SPEC.md#data-types). + +The Code Quality report artifact JSON file must contain an array of objects +with the following properties: | Name | Description | | ---------------------- | -------------------------------------------------------------------------------------- | @@ -63,13 +99,16 @@ Example: ``` NOTE: **Note:** -Although the Code Climate spec supports more properties, those are ignored by GitLab. +Although the Code Climate spec supports more properties, those are ignored by +GitLab. + +## Code Quality reports -For more information on what the Code Quality job should look like, check the -example on [analyzing a project's code quality](../../../ci/examples/code_quality.md). +Once the Code Quality job has completed, GitLab: -GitLab then checks this report, compares the metrics between the source and target -branches, and shows the information right on the merge request. +- Checks the generated report. +- Compares the metrics between the source and target branches. +- Shows the information right on the merge request. If multiple jobs in a pipeline generate a code quality artifact, only the artifact from the last created job (the job with the largest job ID) is used. To avoid confusion, diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6e8aa5d578e..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -55,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 9aea736527b..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entry def tree_entry(repository, sha, path, limit) if use_rugged?(repository, :rugged_tree_entry) - wrap_rugged_call { rugged_tree_entry(repository, sha, path, limit) } + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 29ae9bdd851..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -36,7 +36,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) if use_rugged?(repo, :rugged_find_commit) - wrap_rugged_call { rugged_find(repo, commit_id) } + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -45,7 +45,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) if use_rugged?(repo, :rugged_list_commits_by_oid) - wrap_rugged_call { rugged_batch_by_oid(repo, oids) } + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -68,7 +68,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) if use_rugged?(@repository, :rugged_commit_tree_entry) - wrap_rugged_call { rugged_tree_entry(path) } + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 7bed553393c..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -48,7 +48,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) if use_rugged?(self, :rugged_commit_is_ancestor) - wrap_rugged_call { rugged_is_ancestor?(from, to) } + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 479c5f9d8b7..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entries def tree_entries(repository, sha, path, recursive) if use_rugged?(repository, :rugged_tree_entries) - wrap_rugged_call { tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) } + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index badf943e39c..80b75689334 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -11,17 +11,23 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end - def wrap_rugged_call(&block) + def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time - result = yield + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend duration = Gitlab::Metrics::System.monotonic_time - start if Gitlab::RuggedInstrumentation.active? Gitlab::RuggedInstrumentation.increment_query_count Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) end result diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 70c06e8b308..8bb8c547ae1 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -24,7 +24,24 @@ module Gitlab end def self.active? - Gitlab::SafeRequestStore.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] end end end diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb new file mode 100644 index 00000000000..7b0ab162b15 --- /dev/null +++ b/lib/peek/views/rugged.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Peek + module Views + class Rugged < View + def duration + ::Gitlab::RuggedInstrumentation.query_time + end + + def calls + ::Gitlab::RuggedInstrumentation.query_count + end + + def results + return {} unless calls > 0 + + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + private + + def details + ::Gitlab::RuggedInstrumentation.list_call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def format_call_details(call) + call.merge(duration: (call[:duration] * 1000).round(3), + args: format_args(call[:args])) + end + + def format_args(args) + args.map do |arg| + # Needed to avoid infinite as_json calls + if arg.is_a?(Gitlab::Git::Repository) + arg.to_s + else + arg + end + end + end + + def formatted_duration + ms = duration * 1000 + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end diff --git a/scripts/lint-rugged b/scripts/lint-rugged index e1605b4501b..f40f0489c1a 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -13,9 +13,11 @@ ALLOWED = [ 'lib/gitlab/gitaly_client/storage_settings.rb', # Needed for logging + 'config/initializers/peek.rb', 'config/initializers/lograge.rb', 'lib/gitlab/grape_logging/loggers/perf_logger.rb', - 'lib/gitlab/rugged_instrumentation.rb' + 'lib/gitlab/rugged_instrumentation.rb', + 'lib/peek/views/rugged.rb' ].freeze rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 363ab2c6540..a1f6fb6bc0f 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -201,7 +201,7 @@ function deploy() { HELM_CMD=$(cat << EOF helm upgrade --install \ --wait \ - --timeout 600 \ + --timeout 900 \ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ --set global.appConfig.enableUsagePing=false \ --set global.imagePullPolicy=Always \ diff --git a/spec/controllers/import/github_controller_spec.rb b/spec/controllers/import/github_controller_spec.rb index 059354870b5..5675798ac33 100644 --- a/spec/controllers/import/github_controller_spec.rb +++ b/spec/controllers/import/github_controller_spec.rb @@ -33,6 +33,16 @@ describe Import::GithubController do expect(response).to have_http_status(200) end + + context 'when importing a CI/CD project' do + it 'always prompts for an access token' do + allow(controller).to receive(:github_import_configured?).and_return(true) + + get :new, params: { ci_cd_only: true } + + expect(response).to render_template(:new) + end + end end describe "GET callback" do diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb index c909471bb55..08ac7fb69e7 100644 --- a/spec/factories/lfs_objects.rb +++ b/spec/factories/lfs_objects.rb @@ -14,6 +14,7 @@ FactoryBot.define do # objects, so the test needs to decide which (if any) object gets it trait :correct_oid do oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75' + size 1062 end trait :object_storage do diff --git a/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb index 3991c737a26..0c1be4b4610 100644 --- a/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb @@ -23,18 +23,4 @@ describe Gitlab::Database::Count::ExactCountStrategy do expect(subject).to eq({}) end end - - describe '.enabled?' do - it 'is enabled for PostgreSQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(true) - - expect(described_class.enabled?).to be_truthy - end - - it 'is enabled for MySQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(false) - - expect(described_class.enabled?).to be_truthy - end - end end diff --git a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb index bd3c66d0548..a528707c9dc 100644 --- a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb @@ -48,18 +48,4 @@ describe Gitlab::Database::Count::ReltuplesCountStrategy do end end end - - describe '.enabled?' do - it 'is enabled for PostgreSQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(true) - - expect(described_class.enabled?).to be_truthy - end - - it 'is disabled for MySQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(false) - - expect(described_class.enabled?).to be_falsey - end - end end diff --git a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb index 40d810b195b..a57f033b5ed 100644 --- a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb @@ -56,22 +56,4 @@ describe Gitlab::Database::Count::TablesampleCountStrategy do end end end - - describe '.enabled?' do - before do - stub_feature_flags(tablesample_counts: true) - end - - it 'is enabled for PostgreSQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(true) - - expect(described_class.enabled?).to be_truthy - end - - it 'is disabled for MySQL' do - allow(Gitlab::Database).to receive(:postgresql?).and_return(false) - - expect(described_class.enabled?).to be_falsey - end - end end diff --git a/spec/lib/gitlab/database/count_spec.rb b/spec/lib/gitlab/database/count_spec.rb index 1d096b8fa7c..71d6633f62f 100644 --- a/spec/lib/gitlab/database/count_spec.rb +++ b/spec/lib/gitlab/database/count_spec.rb @@ -9,24 +9,13 @@ describe Gitlab::Database::Count do let(:models) { [Project, Identity] } context '.approximate_counts' do - context 'selecting strategies' do - let(:strategies) { [double('s1', enabled?: true), double('s2', enabled?: false)] } - - it 'uses only enabled strategies' do - expect(strategies[0]).to receive(:new).and_return(double('strategy1', count: {})) - expect(strategies[1]).not_to receive(:new) - - described_class.approximate_counts(models, strategies: strategies) - end - end - context 'fallbacks' do subject { described_class.approximate_counts(models, strategies: strategies) } let(:strategies) do [ - double('s1', enabled?: true, new: first_strategy), - double('s2', enabled?: true, new: second_strategy) + double('s1', new: first_strategy), + double('s2', new: second_strategy) ] end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 41b898df112..dccd50bc472 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -186,6 +186,12 @@ describe Gitlab::Git::Repository, :seed_helper do it { is_expected.to be < 2 } end + describe '#to_s' do + subject { repository.to_s } + + it { is_expected.to eq("<Gitlab::Git::Repository: group/project>") } + end + describe '#object_directory_size' do before do allow(repository.gitaly_repository_client) diff --git a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb index e437647c258..1a4168f7317 100644 --- a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb +++ b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb @@ -16,7 +16,13 @@ describe Gitlab::Git::RuggedImpl::UseRugged, :seed_helper do end subject(:wrapper) do - klazz = Class.new { include Gitlab::Git::RuggedImpl::UseRugged } + klazz = Class.new do + include Gitlab::Git::RuggedImpl::UseRugged + + def rugged_test(ref, test_number) + end + end + klazz.new end @@ -25,6 +31,23 @@ describe Gitlab::Git::RuggedImpl::UseRugged, :seed_helper do Gitlab::GitalyClient.instance_variable_set(:@can_use_disk, {}) end + context '#execute_rugged_call', :request_store do + let(:args) { ['refs/heads/master', 1] } + + before do + allow(Gitlab::RuggedInstrumentation).to receive(:peek_enabled?).and_return(true) + end + + it 'instruments Rugged call' do + expect(subject).to receive(:rugged_test).with(args) + + subject.execute_rugged_call(:rugged_test, args) + + expect(Gitlab::RuggedInstrumentation.query_count).to eq(1) + expect(Gitlab::RuggedInstrumentation.list_call_details.count).to eq(1) + end + end + context 'when feature flag is not persisted' do before do allow(Feature).to receive(:persisted?).with(feature_flag).and_return(false) diff --git a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb index 70eeb9ee66b..2b0bdb909ae 100644 --- a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb @@ -6,6 +6,7 @@ describe Gitlab::ImportExport::LfsRestorer do let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" } let(:project) { create(:project) } let(:shared) { project.import_export_shared } + let(:saver) { Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared) } subject(:restorer) { described_class.new(project: project, shared: shared) } before do @@ -19,49 +20,98 @@ describe Gitlab::ImportExport::LfsRestorer do describe '#restore' do context 'when the archive contains lfs files' do - let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') } - - def create_lfs_object_with_content(content) - dummy_lfs_file = Tempfile.new('existing') - File.write(dummy_lfs_file.path, content) - size = dummy_lfs_file.size - oid = LfsObject.calculate_oid(dummy_lfs_file.path) - LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file) + let(:lfs_object) { create(:lfs_object, :correct_oid, :with_file) } + + # Use the LfsSaver to save data to be restored + def save_lfs_data + %w(project wiki).each do |repository_type| + create( + :lfs_objects_project, + project: project, + repository_type: repository_type, + lfs_object: lfs_object + ) + end + + saver.save + + project.lfs_objects.delete_all end before do - FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path)) - File.write(dummy_lfs_file_path, 'not very large') - allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path]) + save_lfs_data + project.reload end - it 'creates an lfs object for the project' do - expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1) + it 'succeeds' do + expect(restorer.restore).to eq(true) + expect(shared.errors).to be_empty end - it 'assigns the file correctly' do + it 'does not create a new `LfsObject` records, as one already exists' do + expect { restorer.restore }.not_to change { LfsObject.count } + end + + it 'creates new `LfsObjectsProject` records in order to link the project to the existing `LfsObject`' do + expect { restorer.restore }.to change { LfsObjectsProject.count }.by(2) + end + + it 'restores the correct `LfsObject` records' do restorer.restore - expect(project.lfs_objects.first.file.read).to eq('not very large') + expect(project.lfs_objects).to contain_exactly(lfs_object) end - it 'links an existing LFS object if it existed' do - lfs_object = create_lfs_object_with_content('not very large') + it 'restores the correct `LfsObjectsProject` records for the project' do + restorer.restore + expect( + project.lfs_objects_projects.pluck(:repository_type) + ).to contain_exactly('project', 'wiki') + end + + it 'assigns the file correctly' do restorer.restore - expect(project.lfs_objects).to include(lfs_object) + expect(project.lfs_objects.first.file.read).to eq(lfs_object.file.read) end - it 'succeeds' do - expect(restorer.restore).to be_truthy - expect(shared.errors).to be_empty + context 'when there is not an existing `LfsObject`' do + before do + lfs_object.destroy + end + + it 'creates a new lfs object' do + expect { restorer.restore }.to change { LfsObject.count }.by(1) + end + + it 'stores the upload' do + expect_any_instance_of(LfsObjectUploader).to receive(:store!) + + restorer.restore + end end - it 'stores the upload' do - expect_any_instance_of(LfsObjectUploader).to receive(:store!) + context 'when there is no lfs-objects.json file' do + before do + json_file = File.join(shared.export_path, ::Gitlab::ImportExport.lfs_objects_filename) - restorer.restore + FileUtils.rm_rf(json_file) + end + + it 'restores the correct `LfsObject` records' do + restorer.restore + + expect(project.lfs_objects).to contain_exactly(lfs_object) + end + + it 'restores a single `LfsObjectsProject` record for the project with "project" for the `repository_type`' do + restorer.restore + + expect( + project.lfs_objects_projects.pluck(:repository_type) + ).to contain_exactly('project') + end end end diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb index 9b0e21deb2e..c3c88486e16 100644 --- a/spec/lib/gitlab/import_export/lfs_saver_spec.rb +++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb @@ -19,6 +19,11 @@ describe Gitlab::ImportExport::LfsSaver do describe '#save' do context 'when the project has LFS objects locally stored' do let(:lfs_object) { create(:lfs_object, :with_file) } + let(:lfs_json_file) { File.join(shared.export_path, Gitlab::ImportExport.lfs_objects_filename) } + + def lfs_json + JSON.parse(IO.read(lfs_json_file)) + end before do project.lfs_objects << lfs_object @@ -35,6 +40,45 @@ describe Gitlab::ImportExport::LfsSaver do expect(File).to exist("#{shared.export_path}/lfs-objects/#{lfs_object.oid}") end + + describe 'saving a json file' do + before do + # Create two more LfsObjectProject records with different `repository_type`s + %w(wiki design).each do |repository_type| + create( + :lfs_objects_project, + project: project, + repository_type: repository_type, + lfs_object: lfs_object + ) + end + + FileUtils.rm_rf(lfs_json_file) + end + + it 'saves a json file correctly' do + saver.save + + expect(File.exist?(lfs_json_file)).to eq(true) + expect(lfs_json).to eq( + { + lfs_object.oid => [ + LfsObjectsProject.repository_types['wiki'], + LfsObjectsProject.repository_types['design'], + nil + ] + } + ) + end + + it 'does not save a json file if feature is disabled' do + stub_feature_flags(export_lfs_objects_projects: false) + + saver.save + + expect(File.exist?(lfs_json_file)).to eq(false) + end + end end context 'when the LFS objects are stored in object storage' do @@ -42,8 +86,11 @@ describe Gitlab::ImportExport::LfsSaver do before do allow(LfsObjectUploader).to receive(:object_store_enabled?).and_return(true) - allow(lfs_object.file).to receive(:url).and_return('http://my-object-storage.local') project.lfs_objects << lfs_object + + expect_next_instance_of(LfsObjectUploader) do |instance| + expect(instance).to receive(:url).and_return('http://my-object-storage.local') + end end it 'downloads the file to include in an archive' do diff --git a/spec/lib/peek/views/rugged_spec.rb b/spec/lib/peek/views/rugged_spec.rb new file mode 100644 index 00000000000..715b360953c --- /dev/null +++ b/spec/lib/peek/views/rugged_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Peek::Views::Rugged, :request_store do + subject { described_class.new } + + let(:project) { create(:project) } + + before do + allow(Gitlab::RuggedInstrumentation).to receive(:peek_enabled?).and_return(true) + end + + it 'returns no results' do + expect(subject.results).to eq({}) + end + + it 'returns aggregated results' do + ::Gitlab::RuggedInstrumentation.query_time += 1.234 + ::Gitlab::RuggedInstrumentation.increment_query_count + ::Gitlab::RuggedInstrumentation.increment_query_count + + ::Gitlab::RuggedInstrumentation.add_call_details(feature: :rugged_test, + args: [project.repository.raw, 'HEAD'], + duration: 0.123) + ::Gitlab::RuggedInstrumentation.add_call_details(feature: :rugged_test2, + args: [project.repository.raw, 'refs/heads/master'], + duration: 0.456) + + expect(subject.duration).to be_within(0.00001).of(1.234) + expect(subject.calls).to eq(2) + + results = subject.results + expect(results[:calls]).to eq(2) + expect(results[:duration]).to eq('1234.00ms') + expect(results[:details].count).to eq(2) + + expect(results[:details][0][:args]).to eq([project.repository.raw.to_s, "refs/heads/master"]) + expect(results[:details][1][:args]).to eq([project.repository.raw.to_s, "HEAD"]) + end +end |
