diff options
79 files changed, 861 insertions, 676 deletions
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 334eb4b1430..50207a6bb8d 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -287,6 +287,7 @@ .frontend:rules:qa-frontend-node: rules: - <<: *if-master-refs + changes: *frontend-dependency-patterns when: on_success - <<: *if-merge-request changes: *frontend-dependency-patterns diff --git a/.rubocop.yml b/.rubocop.yml index c6046224e4e..b03aa069ebf 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -218,11 +218,9 @@ Gitlab/DuplicateSpecLocation: - ee/spec/lib/gitlab/gl_repository_spec.rb - ee/spec/models/namespace_spec.rb - ee/spec/services/merge_requests/refresh_service_spec.rb - - ee/spec/services/merge_requests/update_service_spec.rb - ee/spec/helpers/ee/auth_helper_spec.rb - ee/spec/models/ee/namespace_spec.rb - ee/spec/services/ee/merge_requests/refresh_service_spec.rb - - ee/spec/services/ee/merge_requests/update_service_spec.rb Cop/InjectEnterpriseEditionModule: Enabled: true @@ -388,11 +386,6 @@ Performance/ChainArrayAllocation: # https://gitlab.com/gitlab-org/gitlab/-/issues/212541 RSpec/RepeatedExample: Exclude: - - 'spec/features/issues/filtered_search/filter_issues_spec.rb' - 'spec/features/merge_request/user_posts_diff_notes_spec.rb' - 'spec/features/projects/files/template_type_dropdown_spec.rb' - - 'spec/finders/environments_finder_spec.rb' - - 'spec/helpers/users_helper_spec.rb' - - 'spec/lib/gitlab/import_export/project/relation_factory_spec.rb' - 'spec/services/notification_service_spec.rb' - - 'spec/services/web_hook_service_spec.rb' @@ -319,7 +319,7 @@ gem 'peek', '~> 1.1' gem 'snowplow-tracker', '~> 0.6.1' # Memory benchmarks -gem 'gitlab-derailed_benchmarks', require: false +gem 'derailed_benchmarks', require: false # Metrics group :metrics do diff --git a/Gemfile.lock b/Gemfile.lock index 7cd5be09e68..c65e2252ce6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,6 +211,17 @@ GEM declarative-option (0.1.0) default_value_for (3.3.0) activerecord (>= 3.2.0, < 6.1) + derailed_benchmarks (1.7.0) + benchmark-ips (~> 2) + get_process_mem (~> 0) + heapy (~> 0) + memory_profiler (~> 0) + mini_histogram (~> 0) + rack (>= 1) + rake (> 10, < 14) + ruby-statistics (>= 2.1) + thor (>= 0.19, < 2) + unicode_plot (>= 0.0.4, < 1.0.0) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) device_detector (1.0.0) @@ -259,6 +270,7 @@ GEM launchy (~> 2.1) mail (~> 2.7) encryptor (3.0.0) + enumerable-statistics (2.0.1) equalizer (0.0.11) erubi (1.9.0) escape_utils (1.2.1) @@ -290,7 +302,7 @@ GEM fast_blank (1.0.0) fast_gettext (1.6.0) ffaker (2.10.0) - ffi (1.11.3) + ffi (1.12.2) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake @@ -371,15 +383,6 @@ GEM github-markup (1.7.0) gitlab-chronic (0.10.5) numerizer (~> 0.2) - gitlab-derailed_benchmarks (1.6.1) - benchmark-ips (~> 2) - get_process_mem (~> 0) - heapy (~> 0) - memory_profiler (~> 0) - rack (>= 1) - rake (> 10, < 14) - ruby-statistics (>= 2.1) - thor (>= 0.19, < 2) gitlab-labkit (0.12.0) actionpack (>= 5.0.0, < 6.1.0) activesupport (>= 5.0.0, < 6.1.0) @@ -632,6 +635,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2019.0331) mimemagic (0.3.3) + mini_histogram (0.1.3) mini_magick (4.9.5) mini_mime (1.0.2) mini_portile2 (2.4.0) @@ -1084,6 +1088,8 @@ GEM unf_ext unf_ext (0.0.7.5) unicode-display_width (1.6.0) + unicode_plot (0.0.4) + enumerable-statistics (>= 2.0.1) unicode_utils (1.4.0) unicorn (5.4.1) kgio (~> 2.6) @@ -1189,6 +1195,7 @@ DEPENDENCIES database_cleaner (~> 1.7.0) deckar01-task_list (= 2.3.1) default_value_for (~> 3.3.0) + derailed_benchmarks device_detector devise (~> 4.6) devise-two-factor (~> 3.1.0) @@ -1231,7 +1238,6 @@ DEPENDENCIES gitaly (~> 12.9.0.pre.rc4) github-markup (~> 1.7.0) gitlab-chronic (~> 0.10.5) - gitlab-derailed_benchmarks gitlab-labkit (= 0.12.0) gitlab-license (~> 1.0) gitlab-mail_room (~> 0.0.3) diff --git a/app/models/concerns/import_state/sidekiq_job_tracker.rb b/app/models/concerns/import_state/sidekiq_job_tracker.rb new file mode 100644 index 00000000000..6bb07b7c06a --- /dev/null +++ b/app/models/concerns/import_state/sidekiq_job_tracker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module ImportState + module SidekiqJobTracker + extend ActiveSupport::Concern + + included do + # Refreshes the expiration time of the associated import job ID. + # + # This method can be used by asynchronous importers to refresh the status, + # preventing the StuckImportJobsWorker from marking the import as failed. + def refresh_jid_expiration + return unless jid + + Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + end + + def self.jid_by(project_id:, status:) + select(:jid).with_status(status).find_by(project_id: project_id) + end + end + end +end diff --git a/app/models/jira_import_state.rb b/app/models/jira_import_state.rb index 0c138d674ea..713feec013f 100644 --- a/app/models/jira_import_state.rb +++ b/app/models/jira_import_state.rb @@ -2,6 +2,7 @@ class JiraImportState < ApplicationRecord include AfterCommitQueue + include ImportState::SidekiqJobTracker self.table_name = 'jira_imports' @@ -66,14 +67,4 @@ class JiraImportState < ApplicationRecord def in_progress? scheduled? || started? end - - def refresh_jid_expiration - return unless jid - - Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - end - - def self.jid_by(project_id:, status:) - select(:jid).with_status(status).find_by(project_id: project_id) - end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 7934b0f8f59..2edb1a45100 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1290,7 +1290,7 @@ class MergeRequest < ApplicationRecord variables.append(key: 'CI_MERGE_REQUEST_PROJECT_URL', value: project.web_url) variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME', value: target_branch.to_s) variables.append(key: 'CI_MERGE_REQUEST_TITLE', value: title) - variables.append(key: 'CI_MERGE_REQUEST_ASSIGNEES', value: assignee_username_list) if assignees.any? + variables.append(key: 'CI_MERGE_REQUEST_ASSIGNEES', value: assignee_username_list) if assignees.present? variables.append(key: 'CI_MERGE_REQUEST_MILESTONE', value: milestone.title) if milestone variables.append(key: 'CI_MERGE_REQUEST_LABELS', value: label_names.join(',')) if labels.present? variables.concat(source_project_variables) diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index cdb034e58fa..52fd23aefd5 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -2,6 +2,7 @@ class ProjectImportState < ApplicationRecord include AfterCommitQueue + include ImportState::SidekiqJobTracker self.table_name = "project_mirror_data" @@ -88,20 +89,6 @@ class ProjectImportState < ApplicationRecord # import? does SQL work so only run it if it looks like there's an import running status == 'started' && project.import? end - - # Refreshes the expiration time of the associated import job ID. - # - # This method can be used by asynchronous importers to refresh the status, - # preventing the StuckImportJobsWorker from marking the import as failed. - def refresh_jid_expiration - return unless jid - - Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - end - - def self.jid_by(project_id:, status:) - select(:jid).with_status(status).find_by(project_id: project_id) - end end ProjectImportState.prepend_if_ee('EE::ProjectImportState') diff --git a/app/policies/snippet_policy.rb b/app/policies/snippet_policy.rb deleted file mode 100644 index 64c56e8091d..00000000000 --- a/app/policies/snippet_policy.rb +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true - -class SnippetPolicy < PersonalSnippetPolicy -end diff --git a/app/services/projects/hashed_storage/base_repository_service.rb b/app/services/projects/hashed_storage/base_repository_service.rb index 09de8d9f0da..d81aa4de9f1 100644 --- a/app/services/projects/hashed_storage/base_repository_service.rb +++ b/app/services/projects/hashed_storage/base_repository_service.rb @@ -40,7 +40,13 @@ module Projects return true end - gitlab_shell.mv_repository(project.repository_storage, from_name, to_name) + gitlab_shell.mv_repository(project.repository_storage, from_name, to_name).tap do |moved| + if moved + logger.info("Repository moved from '#{from_name}' to '#{to_name}' (PROJECT_ID=#{project.id})") + else + logger.error("Repository cannot be moved from '#{from_name}' to '#{to_name}' (PROJECT_ID=#{project.id})") + end + end end def move_repositories diff --git a/app/services/projects/hashed_storage/rollback_service.rb b/app/services/projects/hashed_storage/rollback_service.rb index c437001c440..01b343a12d1 100644 --- a/app/services/projects/hashed_storage/rollback_service.rb +++ b/app/services/projects/hashed_storage/rollback_service.rb @@ -5,6 +5,12 @@ module Projects class RollbackService < BaseService attr_reader :logger, :old_disk_path + def initialize(project, old_disk_path, logger: nil) + @project = project + @old_disk_path = old_disk_path + @logger = logger || Gitlab::AppLogger + end + def execute # Rollback attachments from Hashed Storage to Legacy if project.hashed_storage?(:attachments) diff --git a/changelogs/unreleased/198694-speed-up-new-merge-request-worker.yml b/changelogs/unreleased/198694-speed-up-new-merge-request-worker.yml new file mode 100644 index 00000000000..13d4c85641b --- /dev/null +++ b/changelogs/unreleased/198694-speed-up-new-merge-request-worker.yml @@ -0,0 +1,5 @@ +--- +title: Resolve an N+1 in merge request CI variables +merge_request: 28688 +author: +type: performance diff --git a/changelogs/unreleased/211452-rollback_to_legacy-causes-repos-to-404.yml b/changelogs/unreleased/211452-rollback_to_legacy-causes-repos-to-404.yml new file mode 100644 index 00000000000..73955b4c2c3 --- /dev/null +++ b/changelogs/unreleased/211452-rollback_to_legacy-causes-repos-to-404.yml @@ -0,0 +1,5 @@ +--- +title: Fix storage rollback regression caused by previous refactor +merge_request: 28496 +author: +type: fixed diff --git a/changelogs/unreleased/environment-finder-spec.yml b/changelogs/unreleased/environment-finder-spec.yml new file mode 100644 index 00000000000..31c1bca039f --- /dev/null +++ b/changelogs/unreleased/environment-finder-spec.yml @@ -0,0 +1,5 @@ +--- +title: Fix duplicate spec in environment finder +merge_request: 28857 +author: Rajendra Kadam +type: added diff --git a/changelogs/unreleased/filter-issues.yml b/changelogs/unreleased/filter-issues.yml new file mode 100644 index 00000000000..b29611af5b8 --- /dev/null +++ b/changelogs/unreleased/filter-issues.yml @@ -0,0 +1,5 @@ +--- +title: Fix duplicate spec in filter issues +merge_request: 28860 +author: Rajendra Kadam +type: added diff --git a/changelogs/unreleased/relation-factory-spec.yml b/changelogs/unreleased/relation-factory-spec.yml new file mode 100644 index 00000000000..23b3eacc1b0 --- /dev/null +++ b/changelogs/unreleased/relation-factory-spec.yml @@ -0,0 +1,5 @@ +--- +title: Fix duplicate spec in factory relation spec +merge_request: 28794 +author: Rajendra Kadam +type: added diff --git a/changelogs/unreleased/user-helpers-spec.yml b/changelogs/unreleased/user-helpers-spec.yml new file mode 100644 index 00000000000..60fba3acd26 --- /dev/null +++ b/changelogs/unreleased/user-helpers-spec.yml @@ -0,0 +1,5 @@ +--- +title: Fix duplicate spec from user helper spec +merge_request: 28854 +author: Rajendra Kadam +type: added diff --git a/changelogs/unreleased/validate-dynamic-pipeline-dependencies.yml b/changelogs/unreleased/validate-dynamic-pipeline-dependencies.yml deleted file mode 100644 index 76a75022eab..00000000000 --- a/changelogs/unreleased/validate-dynamic-pipeline-dependencies.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Validate dependency on job generating a CI config when using dynamic child pipelines -merge_request: 27916 -author: -type: added diff --git a/changelogs/unreleased/webhook-service-spec.yml b/changelogs/unreleased/webhook-service-spec.yml new file mode 100644 index 00000000000..294eed58442 --- /dev/null +++ b/changelogs/unreleased/webhook-service-spec.yml @@ -0,0 +1,5 @@ +--- +title: Remove duplicate spec in web hook service spec +merge_request: 28669 +author: Rajendra Kadam +type: fixed diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 62b515f1a3f..a87118a800a 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -48,7 +48,7 @@ The process will execute the following access checks: NOTE: **Note:** In Active Directory, a user is marked as disabled/blocked if the user account control attribute (`userAccountControl:1.2.840.113556.1.4.803`) -has bit 2 set. See <https://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/> +has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> for more information. The user will be set to `ldap_blocked` state in GitLab if the above conditions diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 32062d0767e..0a0e193658e 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -256,9 +256,9 @@ application server, or a Gitaly node. ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('storage_1') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { - 'storage_1' => { + 'praefect' => { 'gitaly-1' => { 'address' => 'tcp://GITALY_HOST:8075', 'token' => 'PRAEFECT_INTERNAL_TOKEN', diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index e317297c854..1c92a429982 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -228,20 +228,8 @@ When disabling `sidekiq_cluster`, you must copy your configuration for `sidekiq_cluster` will be overridden by the options for `sidekiq` when setting `sidekiq['cluster'] = true`. -When using this feature, replace the `sidekiq` service with the -`sidekiq-cluster`service for `gitlab-ctl` commands. - -For example, instead of: - -```shell -sudo gitlab-ctl restart sidekiq -``` - -Use: - -```shell -sudo gitlab-ctl restart sidekiq-cluster -``` +When using this feature, the service called `sidekiq` will now be +running `sidekiq-cluster`. The [concurrency](#managing-concurrency) and other options configured for Sidekiq will be respected. diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index b882595ea4a..0413e5ce953 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -7,7 +7,7 @@ but if they are not available you can still quickly parse ## What is JQ? -As noted in its [manual](https://stedolan.github.io/jq/manual), jq is a command-line JSON processor. The following examples +As noted in its [manual](https://stedolan.github.io/jq/manual/), jq is a command-line JSON processor. The following examples include use cases targeted for parsing GitLab log files. ## Parsing Logs diff --git a/doc/api/projects.md b/doc/api/projects.md index 952d39af8f4..39293b28b79 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -162,6 +162,8 @@ When the user is authenticated and `simple` is not set this returns something li "merge_method": "merge", "autoclose_referenced_issues": true, "suggestion_commit_message": null, + "marked_for_deletion_at": "2020-04-03", + "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, "storage_size": 1038090, @@ -406,6 +408,8 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "merge_method": "merge", "autoclose_referenced_issues": true, "suggestion_commit_message": null, + "marked_for_deletion_at": "2020-04-03", + "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, "storage_size": 1038090, @@ -870,6 +874,8 @@ GET /projects/:id "service_desk_address": null, "autoclose_referenced_issues": true, "suggestion_commit_message": null, + "marked_for_deletion_at": "2020-04-03", + "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, "storage_size": 1038090, diff --git a/doc/api/search.md b/doc/api/search.md index 9ca9cc5c2a4..e1c70fe56a7 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -1,6 +1,6 @@ # Search API -[Introduced][ce-41763] in GitLab 10.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41763) in GitLab 10.5. Every API call to search must be authenticated. @@ -312,7 +312,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: commits **(STARTER)** @@ -383,7 +383,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: users @@ -653,7 +653,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: commits **(STARTER)** @@ -724,7 +724,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: users @@ -1009,7 +1009,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: commits @@ -1082,7 +1082,7 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). ### Scope: users @@ -1104,6 +1104,3 @@ Example response: } ] ``` - -[ce-41763]: https://gitlab.com/gitlab-org/gitlab-foss/issues/41763 -[gitlab-34521]: https://gitlab.com/gitlab-org/gitlab/issues/34521 diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index 2bc897901fa..b39e0b6e540 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -136,11 +136,12 @@ your own script to generate a YAML file, which is then [used to trigger a child This technique can be very powerful in generating pipelines targeting content that changed or to build a matrix of targets and architectures. -In GitLab 12.9, the child pipeline could fail to be created in certain cases, causing the parent pipeline to fail. -This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/209070). - ## Limitations A parent pipeline can trigger many child pipelines, but a child pipeline cannot trigger further child pipelines. See the [related issue](https://gitlab.com/gitlab-org/gitlab/issues/29651) for discussion on possible future improvements. + +When triggering dynamic child pipelines, if the job containing the CI config artifact is not a predecessor of the +trigger job, the child pipeline will fail to be created, causing also the parent pipeline to fail. +In the future we want to validate the trigger job's dependencies [at the time the parent pipeline is created](https://gitlab.com/gitlab-org/gitlab/-/issues/209070) rather than when the child pipeline is created. diff --git a/doc/development/documentation/site_architecture/release_process.md b/doc/development/documentation/site_architecture/release_process.md index 45cf1a5ee76..a8da565124b 100644 --- a/doc/development/documentation/site_architecture/release_process.md +++ b/doc/development/documentation/site_architecture/release_process.md @@ -91,8 +91,9 @@ this needs to happen when the stable branches for all products have been created Once you push, the `image:docker-singe` job will create a new Docker image tagged with the branch name you created in the first step. In the end, the image will be uploaded in the [Container Registry](https://gitlab.com/gitlab-org/gitlab-docs/container_registry) - and it will be listed under the - [`registry` environment folder](https://gitlab.com/gitlab-org/gitlab-docs/-/environments/folders/registry). + and it will be listed under the `registry` environment folder at + `https://gitlab.com/gitlab-org/gitlab-docs/-/environments/folders/registry` (must + have developer access). Optionally, you can test locally by building the image and running it: diff --git a/doc/development/licensing.md b/doc/development/licensing.md index c7676cc2596..06a71967d06 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -1,10 +1,10 @@ # GitLab Licensing and Compatibility -[GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss/) (CE) is licensed [under the terms of the MIT License][CE]. [GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/) (EE) is licensed under "[The GitLab Enterprise Edition (EE) license][EE]" wherein there are more restrictions. +[GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss/) (CE) is licensed [under the terms of the MIT License](https://gitlab.com/gitlab-org/gitlab-foss/blob/master/LICENSE). [GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/) (EE) is licensed under "[The GitLab Enterprise Edition (EE) license](https://gitlab.com/gitlab-org/gitlab/blob/master/LICENSE)" wherein there are more restrictions. ## Automated Testing -In order to comply with the terms the libraries we use are licensed under, we have to make sure to check new gems for compatible licenses whenever they're added. To automate this process, we use the [license_finder][license_finder] gem by Pivotal. It runs every time a new commit is pushed and verifies that all gems and node modules in the bundle use a license that doesn't conflict with the licensing of either GitLab Community Edition or GitLab Enterprise Edition. +In order to comply with the terms the libraries we use are licensed under, we have to make sure to check new gems for compatible licenses whenever they're added. To automate this process, we use the [license_finder](https://github.com/pivotal/LicenseFinder) gem by Pivotal. It runs every time a new commit is pushed and verifies that all gems and node modules in the bundle use a license that doesn't conflict with the licensing of either GitLab Community Edition or GitLab Enterprise Edition. There are some limitations with the automated testing, however. CSS, JavaScript, or Ruby libraries which are not included by way of Bundler, NPM, or Yarn (for instance those manually copied into our source tree in the `vendor` directory), must be verified manually and independently. Take care whenever one such library is used, as automated tests won't catch problematic licenses from them. @@ -40,7 +40,7 @@ license_finder licenses add my_unknown_dependency MIT For all of the above, please include `--why "Reason"` and `--who "My Name"` so the `decisions.yml` file can keep track of when, why, and who approved of a dependency. -More detailed information on how the gem and its commands work is available in the [License Finder README][license_finder]. +More detailed information on how the gem and its commands work is available in the [License Finder README](https://github.com/pivotal/LicenseFinder). ## Acceptable Licenses @@ -49,24 +49,24 @@ Libraries with the following licenses are acceptable for use: - [MIT License](https://choosealicense.com/licenses/mit/) (the MIT Expat License specifically): The MIT License requires that the license itself is included with all copies of the source. It is a permissive (non-copyleft) license as defined by the Open Source Initiative. - [GNU Lesser General Public License (GNU LGPL)](https://choosealicense.com/licenses/lgpl-3.0/) (version 2, version 3): GPL constraints regarding modification and redistribution under the same license are not required of projects using an LGPL library, only upon modification of the LGPL-licensed library itself. - [Apache 2.0 License](https://choosealicense.com/licenses/apache-2.0/): A permissive license that also provides an express grant of patent rights from contributors to users. -- [Ruby 1.8 License][ruby-1.8]: Dual-licensed under either itself or the GPLv2, defer to the Ruby License itself. Acceptable because of point 3b: "You may distribute the software in object code or binary form, provided that you do at least ONE of the following: b) accompany the distribution with the machine-readable source of the software." -- [Ruby 1.9 License][ruby-1.9]: Dual-licensed under either itself or the BSD 2-Clause License, defer to BSD 2-Clause. -- [BSD 2-Clause License][BSD-2-Clause]: A permissive (non-copyleft) license as defined by the Open Source Initiative. -- [BSD 3-Clause License][BSD-3-Clause] (also known as New BSD or Modified BSD): A permissive (non-copyleft) license as defined by the Open Source Initiative -- [ISC License][ISC] (also known as the OpenBSD License): A permissive (non-copyleft) license as defined by the Open Source Initiative. -- [Creative Commons Zero (CC0)][CC0]: A public domain dedication, recommended as a way to disclaim copyright on your work to the maximum extent possible. -- [Unlicense][UNLICENSE]: Another public domain dedication. -- [OWFa 1.0][OWFa1]: An open-source license and patent grant designed for specifications. +- [Ruby 1.8 License](https://github.com/ruby/ruby/blob/ruby_1_8_6/COPYING): Dual-licensed under either itself or the GPLv2, defer to the Ruby License itself. Acceptable because of point 3b: "You may distribute the software in object code or binary form, provided that you do at least ONE of the following: b) accompany the distribution with the machine-readable source of the software." +- [Ruby 1.9 License](https://www.ruby-lang.org/en/about/license.txt): Dual-licensed under either itself or the BSD 2-Clause License, defer to BSD 2-Clause. +- [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause): A permissive (non-copyleft) license as defined by the Open Source Initiative. +- [BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) (also known as New BSD or Modified BSD): A permissive (non-copyleft) license as defined by the Open Source Initiative +- [ISC License](https://opensource.org/licenses/ISC) (also known as the OpenBSD License): A permissive (non-copyleft) license as defined by the Open Source Initiative. +- [Creative Commons Zero (CC0)](https://creativecommons.org/publicdomain/zero/1.0/): A public domain dedication, recommended as a way to disclaim copyright on your work to the maximum extent possible. +- [Unlicense](https://unlicense.org): Another public domain dedication. +- [OWFa 1.0](http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0): An open-source license and patent grant designed for specifications. - [JSON License](https://www.json.org/license.html): Equivalent to the MIT license plus the statement, "The Software shall be used for Good, not Evil." ## Unacceptable Licenses Libraries with the following licenses require legal approval for use: -- [GNU GPL](https://choosealicense.com/licenses/gpl-3.0/) (version 1, [version 2][GPLv2], [version 3][GPLv3], or any future versions): GPL-licensed libraries cannot be linked to from non-GPL projects. +- [GNU GPL](https://choosealicense.com/licenses/gpl-3.0/) (version 1, [version 2](http://www.gnu.org/licenses/gpl-2.0.txt), [version 3](http://www.gnu.org/licenses/gpl-3.0.txt), or any future versions): GPL-licensed libraries cannot be linked to from non-GPL projects. - [GNU AGPLv3](https://choosealicense.com/licenses/agpl-3.0/): AGPL-licensed libraries cannot be linked to from non-GPL projects. -- [Open Software License (OSL)][OSL]: is a copyleft license. In addition, the FSF [recommend against its use][OSL-GNU]. -- [WTFPL][WTFPL]: is a public domain dedication [rejected by the OSI (3.2)][WTFPL-OSI]. Also has a strong language which is not in accordance with our diversity policy. +- [Open Software License (OSL)](https://opensource.org/licenses/OSL-3.0): is a copyleft license. In addition, the FSF [recommend against its use](https://www.gnu.org/licenses/license-list.en.html#OSL). +- [WTFPL](https://wtfpl.net): is a public domain dedication [rejected by the OSI (3.2)](https://opensource.org/minutes20090304). Also has a strong language which is not in accordance with our diversity policy. ## GPL Cooperation Commitment @@ -90,13 +90,13 @@ GitLab means GitLab Inc. and its affiliates and subsidiaries. ## Requesting Approval for Licenses or any other Intellectual Property -Libraries that are not already approved and listed on the [Acceptable Licenses][Acceptable-Licenses] list or that may be listed on the [Unacceptable Licenses][Unacceptable-Licenses] list may be submitted to the legal team for review and use on a case-by-case basis. Please email `legal@gitlab.com` with the details of how the software will be used, whether or not it will be modified, and how it will be distributed (if at all). After a decision has been made, the original requestor is responsible for updating this document, if applicable. Not all approvals will be approved for universal use and may continue to remain on the Unacceptable License list. +Libraries that are not already approved and listed on the [Acceptable Licenses](#acceptable-licenses) list or that may be listed on the [Unacceptable Licenses](#unacceptable-licenses) list may be submitted to the legal team for review and use on a case-by-case basis. Please email `legal@gitlab.com` with the details of how the software will be used, whether or not it will be modified, and how it will be distributed (if at all). After a decision has been made, the original requestor is responsible for updating this document, if applicable. Not all approvals will be approved for universal use and may continue to remain on the Unacceptable License list. All inquiries relating to patents should be directed to the Legal team. ## Notes -Decisions regarding the GNU GPL licenses are based on information provided by [The GNU Project][GNU-GPL-FAQ], as well as [the Open Source Initiative][OSI-GPL], which both state that linking GPL libraries makes the program itself GPL. +Decisions regarding the GNU GPL licenses are based on information provided by [The GNU Project](http://www.gnu.org/licenses/gpl-faq.html#IfLibraryIsGPL), as well as [the Open Source Initiative](https://opensource.org/faq#linking-proprietary-code), which both state that linking GPL libraries makes the program itself GPL. If a gem uses a license which is not listed above, open an issue and ask. If a license is not included in the "acceptable" list, operate under the assumption that it is not acceptable. @@ -105,27 +105,3 @@ Keep in mind that each license has its own restrictions (typically defined in th Dependencies which are only used in development or test environment are exempt from license requirements, as they're not distributed for use in production. **NOTE:** This document is **not** legal advice, nor is it comprehensive. It should not be taken as such. - -[CE]: https://gitlab.com/gitlab-org/gitlab-foss/blob/master/LICENSE -[EE]: https://gitlab.com/gitlab-org/gitlab/blob/master/LICENSE -[license_finder]: https://github.com/pivotal/LicenseFinder -[ruby-1.8]: https://github.com/ruby/ruby/blob/ruby_1_8_6/COPYING -[ruby-1.9]: https://www.ruby-lang.org/en/about/license.txt -[BSD-2-Clause]: https://opensource.org/licenses/BSD-2-Clause -[BSD-3-Clause]: https://opensource.org/licenses/BSD-3-Clause -[ISC]: https://opensource.org/licenses/ISC -[CC0]: https://creativecommons.org/publicdomain/zero/1.0/ -[GPLv2]: http://www.gnu.org/licenses/gpl-2.0.txt -[GPLv3]: http://www.gnu.org/licenses/gpl-3.0.txt -[GNU-GPL-FAQ]: http://www.gnu.org/licenses/gpl-faq.html#IfLibraryIsGPL -[OSI-GPL]: https://opensource.org/faq#linking-proprietary-code -[OSL]: https://opensource.org/licenses/OSL-3.0 -[OSL-GNU]: https://www.gnu.org/licenses/license-list.en.html#OSL -[Org-Repo]: https://gitlab.com/gitlab-com/organization -[UNLICENSE]: https://unlicense.org -[OWFa1]: http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0 -[x-list]: https://www.apache.org/legal/resolved.html#category-x -[Acceptable-Licenses]: #acceptable-licenses -[Unacceptable-Licenses]: #unacceptable-licenses -[WTFPL]: https://wtfpl.net -[WTFPL-OSI]: https://opensource.org/minutes20090304 diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index 521223e626b..d851b9f5dc7 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -81,13 +81,10 @@ application. 1. Change `YOUR_AUTH0_CLIENT_SECRET` to the client secret from the Auth0 Console page from step 5. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be an Auth0 icon below the regular sign in form. Click the icon to begin the authentication process. Auth0 will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/azure.md b/doc/integration/azure.md index d6205b60f3d..75469fcbb98 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -81,10 +81,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Microsoft icon below the regular sign in form. Click the icon to begin the authentication process. Microsoft will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index ac462ff7bec..dc5c7a138f8 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -118,8 +118,8 @@ you to use. from the Bitbucket application page. 1. Save the configuration file. -1. For the changes to take effect, [reconfigure GitLab][] if you installed via - Omnibus, or [restart][] if installed from source. +1. For the changes to take effect, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) if you installed via + Omnibus, or [restart](../administration/restart_gitlab.md#installations-from-source) if installed from source. On the sign in page there should now be a Bitbucket icon below the regular sign in form. Click the icon to begin the authentication process. Bitbucket will ask @@ -129,11 +129,7 @@ well, the user will be returned to GitLab and will be signed in. ## Bitbucket project import Once the above configuration is set up, you can use Bitbucket to sign into -GitLab and [start importing your projects][bb-import]. +GitLab and [start importing your projects](../user/project/import/bitbucket.md). If you want to import projects from Bitbucket, but don't want to enable signing in, you can [disable Sign-Ins in the admin panel](omniauth.md#enable-or-disable-sign-in-with-an-omniauth-provider-without-disabling-import-sources). - -[bb-import]: ../user/project/import/bitbucket.md -[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/cas.md b/doc/integration/cas.md index cb92198f88d..cf4e501e772 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -57,10 +57,8 @@ To enable the CAS OmniAuth provider you must register your application with your 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or + [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to + take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a CAS tab in the sign in form. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index 97623b2a782..fc65191994d 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -92,10 +92,7 @@ To enable the Facebook OmniAuth provider you must register your application with 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Facebook icon below the regular sign in form. Click the icon to begin the authentication process. Facebook will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 36810e7c513..4e62f6389c8 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -75,12 +75,9 @@ GitLab.com will generate an application ID and secret key for you to use. 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a GitLab.com icon below the regular sign in form. Click the icon to begin the authentication process. GitLab.com will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to your GitLab instance and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/google.md b/doc/integration/google.md index ee7e7fd6193..0f848bbc7aa 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -108,13 +108,10 @@ On your GitLab server: ``` 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Google icon below the regular sign in form. Click the icon to begin the authentication process. Google will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 8b83f885fdd..7753629c371 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -1,9 +1,9 @@ # GitLab Jira development panel integration **(PREMIUM)** -> [Introduced][ee-2381] in [GitLab Premium][eep] 10.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2381) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. -Complementary to our [existing Jira][existing-jira] project integration, you're now able to integrate -GitLab projects with [Jira Development Panel][jira-development-panel]. Both can be used +Complementary to our [existing Jira](../user/project/integrations/jira.md) project integration, you're now able to integrate +GitLab projects with [Jira Development Panel](https://confluence.atlassian.com/adminjiraserver070/). Both can be used simultaneously. This works with self-managed GitLab or GitLab.com integrated with: - Jira hosted by you. @@ -17,7 +17,7 @@ as well as projects of the top-level group's subgroups nesting down, are connect a GitLab personal namespace in the Jira configuration, which will then connect the projects in that personal namespace to Jira. NOTE: **Note**: -Note this is different from the [existing Jira][existing-jira] project integration, where the mapping +Note this is different from the [existing Jira](../user/project/integrations/jira.md) project integration, where the mapping is one GitLab project to the entire Jira instance. We recommend that a GitLab group admin @@ -164,7 +164,7 @@ Click the links to see your GitLab repository data. ## Limitations -- This integration is currently not supported on GitLab instances under a [relative url][relative-url] (e.g. `http://example.com/gitlab`). +- This integration is currently not supported on GitLab instances under a [relative url](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab) (for example, `http://example.com/gitlab`). ## Changelog @@ -175,9 +175,3 @@ Click the links to see your GitLab repository data. ### 11.1 - [Support GitLab subgroups in Jira development panel](https://gitlab.com/gitlab-org/gitlab/issues/3561) - -[existing-jira]: ../user/project/integrations/jira.md -[jira-development-panel]: https://confluence.atlassian.com/adminjiraserver070/integrating-with-development-tools-776637096.html#Integratingwithdevelopmenttools-Developmentpanelonissues -[eep]: https://about.gitlab.com/pricing/ -[ee-2381]: https://gitlab.com/gitlab-org/gitlab/issues/2381 -[relative-url]: https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index b77f7a952aa..01bd2a8c0a0 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -1,18 +1,18 @@ # Kerberos integration **(STARTER ONLY)** -GitLab can integrate with [Kerberos][kerb] as an authentication mechanism. +GitLab can integrate with [Kerberos](https://web.mit.edu/kerberos/) as an authentication mechanism. ## Overview -[Kerberos][kerb] is a secure method for authenticating a request for a service in a +[Kerberos](https://web.mit.edu/kerberos/) is a secure method for authenticating a request for a service in a computer network. Kerberos was developed in the Athena Project at the -[Massachusetts Institute of Technology (MIT)][mit]. The name is taken from Greek +[Massachusetts Institute of Technology (MIT)](http://web.mit.edu/). The name is taken from Greek mythology; Kerberos was a three-headed dog who guarded the gates of Hades. ## Use-cases - GitLab can be configured to allow your users to sign with their Kerberos credentials. -- You can use Kerberos to [prevent][why-kerb] anyone from intercepting or eavesdropping on the transmitted password. +- You can use Kerberos to [prevent](http://web.mit.edu/sipb/doc/working/guide/guide/node20.html) anyone from intercepting or eavesdropping on the transmitted password. ## Configuration @@ -65,7 +65,7 @@ For source installations, make sure the `kerberos` gem group keytab: /etc/http.keytab ``` -1. [Restart GitLab] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. **Omnibus package installations** @@ -78,7 +78,7 @@ For source installations, make sure the `kerberos` gem group gitlab_rails['kerberos_keytab'] = "/etc/http.keytab" ``` -1. [Reconfigure GitLab] for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. GitLab will now offer the `negotiate` authentication method for signing in and HTTP Git access, enabling Git clients that support this authentication protocol @@ -165,7 +165,7 @@ keep offering only `basic` authentication. https: true ``` -1. [Restart GitLab] and NGINX for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) and NGINX for the changes to take effect. **For Omnibus package installations** @@ -177,7 +177,7 @@ keep offering only `basic` authentication. gitlab_rails['kerberos_https'] = true ``` -1. [Reconfigure GitLab] for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. After this change, all Git remote URLs will have to be updated to `https://gitlab.example.com:8443/mygroup/myproject.git` in order to use @@ -212,7 +212,7 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / - { name: 'kerberos' } # <-- remove this line ``` -1. [Restart GitLab] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. **For Omnibus installations** @@ -225,7 +225,7 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / ] ``` -1. [Reconfigure GitLab] for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ## Support for Active Directory Kerberos environments @@ -233,7 +233,7 @@ When using Kerberos ticket-based authentication in an Active Directory domain, it may be necessary to increase the maximum header size allowed by NGINX, as extensions to the Kerberos protocol may result in HTTP authentication headers larger than the default size of 8kB. Configure `large_client_header_buffers` -to a larger value in [the NGINX configuration][nginx]. +to a larger value in [the NGINX configuration](http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers). ## Troubleshooting @@ -293,11 +293,3 @@ See also: [Git v2.11 release notes](https://github.com/git/git/blob/master/Docum - <https://help.ubuntu.com/community/Kerberos> - <http://blog.manula.org/2012/04/setting-up-kerberos-server-with-debian.html> - <https://www.roguelynn.com/words/explain-like-im-5-kerberos/> - -[restart gitlab]: ../administration/restart_gitlab.md#installations-from-source -[reconfigure gitlab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[nginx]: http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers -[kerb]: https://web.mit.edu/kerberos/ -[mit]: http://web.mit.edu/ -[why-kerb]: http://web.mit.edu/sipb/doc/working/guide/guide/node20.html -[ee]: https://about.gitlab.com/pricing/ diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index 7fd96d727b5..8566134815a 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -51,7 +51,7 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings -1. Add the provider-specific configuration for your provider, as [described in the gem's README][1] +1. Add the provider-specific configuration for your provider, as [described in the gem's README](https://gitlab.com/satorix/omniauth-oauth2-generic#gitlab-config-example) 1. Save the configuration file @@ -61,5 +61,3 @@ On the sign in page there should now be a new button below the regular sign in f Click the button to begin your provider's authentication process. This will direct the browser to your OAuth2 Provider's authentication page. If everything goes well the user will be returned to your GitLab instance and will be signed in. - -[1]: https://gitlab.com/satorix/omniauth-oauth2-generic#gitlab-config-example diff --git a/doc/integration/openid_connect_provider.md b/doc/integration/openid_connect_provider.md index f75630b93a2..3da17347e91 100644 --- a/doc/integration/openid_connect_provider.md +++ b/doc/integration/openid_connect_provider.md @@ -16,13 +16,13 @@ mobile applications. On the client side, you can use [OmniAuth::OpenIDConnect](https://github.com/jjbohn/omniauth-openid-connect/) for Rails applications, or any of the other available [client implementations](https://openid.net/developers/libraries/#connect). -GitLab's implementation uses the [doorkeeper-openid_connect] gem, refer +GitLab's implementation uses the [doorkeeper-openid_connect](https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website") gem, refer to its README for more details about which parts of the specifications are supported. ## Enabling OpenID Connect for OAuth applications -Refer to the [OAuth guide] for basic information on how to set up OAuth +Refer to the [OAuth guide](oauth_provider.md) for basic information on how to set up OAuth applications in GitLab. To enable OIDC for an application, all you have to do is select the `openid` scope in the application settings. @@ -45,6 +45,3 @@ Currently the following user information is shared with clients: | `groups` | `array` | Names of the groups the user is a member of Only the `sub` and `sub_legacy` claims are included in the ID token, all other claims are available from the `/oauth/userinfo` endpoint used by OIDC clients. - -[doorkeeper-openid_connect]: https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website" -[OAuth guide]: oauth_provider.md "GitLab as OAuth2 authentication service provider" diff --git a/doc/integration/saml.md b/doc/integration/saml.md index b4f5bfda8c5..47ffe333c0f 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -120,7 +120,7 @@ in your SAML IdP: 1. Change the value of `issuer` to a unique name, which will identify the application to the IdP. -1. For the changes to take effect, you must [reconfigure][] GitLab if you installed via Omnibus or [restart GitLab][] if you installed from source. +1. For the changes to take effect, you must [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab if you installed via Omnibus or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) if you installed from source. 1. Register the GitLab SP in your SAML 2.0 IdP, using the application name specified in `issuer`. @@ -294,7 +294,7 @@ If you want some SAML authentication methods to count as 2FA on a per session ba ] ``` -1. Save the file and [reconfigure][] GitLab for the changes to take effect. +1. Save the file and [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. --- @@ -324,7 +324,7 @@ If you want some SAML authentication methods to count as 2FA on a per session ba } ``` -1. Save the file and [restart GitLab][] for the changes ot take effect +1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes ot take effect In addition to the changes in GitLab, make sure that your Idp is returning the `AuthnContext`. For example: @@ -630,6 +630,3 @@ For this you need take the following into account: Make sure that one of the above described scenarios is valid, or the requests will fail with one of the mentioned errors. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/trello_power_up.md b/doc/integration/trello_power_up.md index 834d63d1166..fc55dbb9654 100644 --- a/doc/integration/trello_power_up.md +++ b/doc/integration/trello_power_up.md @@ -36,7 +36,5 @@ from Trello. your user **Settings** > **Access Tokens**. Learn more about generating a personal access token in the -[Personal Access Token Documentation][personal-access-token-documentation]. +[Personal Access Token Documentation](../user/profile/personal_access_tokens.md). Don't forget to check the API scope checkbox! - -[personal-access-token-documentation]: ../user/profile/personal_access_tokens.md diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index 8056c672d67..95de56f24d8 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -75,10 +75,7 @@ To enable the Twitter OmniAuth provider you must register your application with 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Twitter icon below the regular sign in form. Click the icon to begin the authentication process. Twitter will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. - -[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 723f4b8acbc..b4fdf307d8b 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -62,7 +62,7 @@ The following languages and dependency managers are supported. | PHP ([Composer](https://getcomposer.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Python ([pip](https://pip.pypa.io/en/stable/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Python ([Pipfile](https://pipenv.kennethreitz.org/en/latest/basics/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/11756 "Pipfile.lock support for Dependency Scanning"))| not available | -| Python ([poetry](http://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available | +| Python ([poetry](https://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available | | Ruby ([gem](https://rubygems.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | | Scala ([sbt](https://www.scala-sbt.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Go ([Go Modules](https://github.com/golang/go/wiki/Modules)) | yes ([alpha](https://gitlab.com/gitlab-org/gitlab/issues/7132)) | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index c968ad26b5e..ef20e074b3e 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -60,7 +60,7 @@ The following languages and package managers are supported. | Elixir | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)| | C++/C | [conan](https://conan.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| | Scala | [sbt](https://www.scala-sbt.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| -| Rust | [cargo](https://crates.io) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| +| Rust | [cargo](https://crates.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| | PHP | [composer](https://getcomposer.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| ## Requirements diff --git a/doc/user/markdown.md b/doc/user/markdown.md index d297a704925..ca5347ca613 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -995,7 +995,7 @@ Here's a sample audio clip: You can also use raw HTML in your Markdown, and it will usually work pretty well. -See the documentation for HTML::Pipeline's [SanitizationFilter](https://www.rubydoc.info/gems/html-pipeline/1.11.0/HTML/Pipeline/SanitizationFilter#WHITELIST-constant) +See the documentation for HTML::Pipeline's [SanitizationFilter](https://github.com/jch/html-pipeline/blob/v2.12.3/lib/html/pipeline/sanitization_filter.rb#L42) class for the list of allowed HTML tags and attributes. In addition to the default `SanitizationFilter` whitelist, GitLab allows `span`, `abbr`, `details` and `summary` elements. diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 4bc44d1d7d8..1ef2f593621 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -44,12 +44,12 @@ That's all! You can now start using the Slack slash commands. ## Usage After confirming the installation, you, and everyone else in your Slack team, -can use all the [slash commands]. +can use all the [slash commands](../../../integration/slash_commands.md). When you perform your first slash command you will be asked to authorize your Slack user on GitLab.com. -The only difference with the [manually configurable Slack slash commands][slack-manual] +The only difference with the [manually configurable Slack slash commands](slack_slash_commands.md) is that all the commands should be prefixed with the `/gitlab` keyword. We are working on making this configurable in the future. @@ -59,6 +59,3 @@ project, you would do: ```plaintext /gitlab gitlab-org/gitlab issue show 1001 ``` - -[slash commands]: ../../../integration/slash_commands.md -[slack-manual]: slack_slash_commands.md diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 21df15c90f4..75565dd2750 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -2,7 +2,7 @@ You can find the available integrations under your project's **Settings ➔ Integrations** page. You need to have at least -[maintainer permission][permissions] on the project. +[maintainer permission](../../permissions.md) on the project. ## Integrations @@ -20,5 +20,3 @@ like pushes, issues or merge requests. GitLab will send a POST request with data to the webhook URL. [Learn more about webhooks.](webhooks.md) - -[permissions]: ../../permissions.md diff --git a/doc/user/project/integrations/jira.md b/doc/user/project/integrations/jira.md index cef9300fc67..442f3229de2 100644 --- a/doc/user/project/integrations/jira.md +++ b/doc/user/project/integrations/jira.md @@ -37,7 +37,7 @@ configured. Therefore, you will not have to explicitly associate a GitLab project with any single Jira project. If you have one Jira instance, you can pre-fill the settings page with a default -template. See the [Services Templates][services-templates] docs. +template. See the [Services Templates](services_templates.md) docs. In order to enable the Jira service in GitLab, you need to first configure the project in Jira and then enter the correct values in GitLab. @@ -55,7 +55,7 @@ In order to enable the Jira service in GitLab, you need to first configure the p > **Notes:** > -> - The currently supported Jira versions are `v6.x` and `v7.x.`. GitLab 7.8 or +> - The currently supported Jira versions are `v6.x, v7.x, v8.x` . GitLab 7.8 or > higher is required. > - GitLab 8.14 introduced a new way to integrate with Jira which greatly simplified > the configuration options you have to enter. If you are using an older version, @@ -212,5 +212,3 @@ which may lead to a `401 unauthorized` error when testing your Jira integration. If CAPTCHA has been triggered, you will not be able to use Jira's REST API to authenticate with the Jira site. You will need to log in to your Jira instance and complete the CAPTCHA. - -[services-templates]: services_templates.md diff --git a/doc/user/project/integrations/mattermost_slash_commands.md b/doc/user/project/integrations/mattermost_slash_commands.md index 35bcc907185..6abd613a019 100644 --- a/doc/user/project/integrations/mattermost_slash_commands.md +++ b/doc/user/project/integrations/mattermost_slash_commands.md @@ -15,7 +15,7 @@ Mattermost 3.4 and up is required. If you have the Omnibus GitLab package installed, Mattermost is already bundled in it. All you have to do is configure it. Read more in the -[Omnibus GitLab Mattermost documentation][omnimmdocs]. +[Omnibus GitLab Mattermost documentation](https://docs.gitlab.com/omnibus/gitlab-mattermost/). ## Automated Configuration @@ -133,7 +133,7 @@ The available slash commands are: | ------- | ----------- | ------- | | <kbd>/<trigger> issue new <title> <kbd>⇧ Shift</kbd>+<kbd>↵ Enter</kbd> <description></kbd> | Create a new issue in the project that `<trigger>` is tied to. `<description>` is optional. | `/gitlab issue new We need to change the homepage` | | <kbd>/<trigger> issue show <issue-number></kbd> | Show the issue with ID `<issue-number>` from the project that `<trigger>` is tied to. | `/gitlab issue show 42` | -| <kbd>/<trigger> deploy <environment> to <environment></kbd> | Start the CI job that deploys from one environment to another, for example `staging` to `production`. CI/CD must be [properly configured][ciyaml]. | `/gitlab deploy staging to production` | +| <kbd>/<trigger> deploy <environment> to <environment></kbd> | Start the CI job that deploys from one environment to another, for example `staging` to `production`. CI/CD must be [properly configured](../../../ci/yaml/README.md). | `/gitlab deploy staging to production` | To see a list of available commands to interact with GitLab, type the trigger word followed by <kbd>help</kbd>. Example: `/gitlab help` @@ -147,9 +147,5 @@ the [permissions you have on the project](../../permissions.md#project-members-p ## Further reading -- [Mattermost slash commands documentation][mmslashdocs] -- [Omnibus GitLab Mattermost][omnimmdocs] - -[omnimmdocs]: https://docs.gitlab.com/omnibus/gitlab-mattermost/ -[mmslashdocs]: https://docs.mattermost.com/developer/slash-commands.html -[ciyaml]: ../../../ci/yaml/README.md +- [Mattermost slash commands documentation](https://docs.mattermost.com/developer/slash-commands.html) +- [Omnibus GitLab Mattermost](https://docs.gitlab.com/omnibus/gitlab-mattermost/) diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 2b1608ad60c..7a21887d29f 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -1,8 +1,8 @@ # Prometheus integration -> [Introduced][ce-8935] in GitLab 9.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935) in GitLab 9.0. -GitLab offers powerful integration with [Prometheus] for monitoring key metrics of your apps, directly within GitLab. +GitLab offers powerful integration with [Prometheus](https://prometheus.io) for monitoring key metrics of your apps, directly within GitLab. Metrics for each environment are retrieved from Prometheus, and then displayed within the GitLab interface. @@ -727,7 +727,7 @@ If the metric exceeds the threshold of the alert for over 5 minutes, an email wi ## Determining the performance impact of a merge -> - [Introduced][ce-10408] in GitLab 9.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408) in GitLab 9.2. > - GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-foss/issues/27439) of the 30 minute averages. Developers can view the performance impact of their changes within the merge @@ -755,7 +755,7 @@ Prometheus server. ### Embedding GitLab-managed Kubernetes metrics -> [Introduced][ce-29691] in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. It is possible to display metrics charts within [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm) fields such as issue or merge request descriptions. The maximum number of embedded charts allowed in a GitLab Flavored Markdown field is 100. @@ -895,18 +895,3 @@ If the "No data found" screen continues to appear, it could be due to: [run a query](prometheus_library/kubernetes.md#metrics-supported), replacing `$CI_ENVIRONMENT_SLUG` with the name of your environment. - You may need to re-add the GitLab predefined common metrics. This can be done by running the [import common metrics Rake task](../../../administration/raketasks/maintenance.md#import-common-metrics). - -[autodeploy]: ../../../topics/autodevops/index.md#auto-deploy -[kubernetes]: https://kubernetes.io -[kube]: ./kubernetes.md -[prometheus-k8s-sd]: https://prometheus.io/docs/operating/configuration/#<kubernetes_sd_config> -[prometheus]: https://prometheus.io -[gitlab-prometheus-k8s-monitor]: ../../../administration/monitoring/prometheus/index.md#configuring-prometheus-to-monitor-kubernetes -[prometheus-docker-image]: https://hub.docker.com/r/prom/prometheus/ -[prometheus-yml]:samples/prometheus.yml -[gitlab.com-ip-range]: https://gitlab.com/gitlab-com/infrastructure/issues/434 -[ci-environment-slug]: ../../../ci/variables/#predefined-environment-variables -[ce-8935]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935 -[ce-10408]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408 -[ce-29691]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691 -[promgldocs]: ../../../administration/monitoring/prometheus/index.md diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 0fbf5ec6f4b..ebb78d5c54a 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -126,6 +126,8 @@ to help summarize changes between versions. Designs can be explored in greater detail by zooming in and out of the image. Control the amount of zoom with the `+` and `-` buttons at the bottom of the image. While zoomed, you can still [start new discussions](#starting-discussions-on-designs) on the image, and see any existing ones. +[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197324) in GitLab 12.10, while zoomed in, +you can click-and-drag on the image to move around it.  diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index fe7c2c2757b..71fbdaf112f 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -34,7 +34,7 @@ for other languages have plugins to add support for it, like: Other coverage analysis frameworks support the format out of the box, for example: - [Istanbul](https://istanbul.js.org/docs/advanced/alternative-reporters/#cobertura) (JavaScript) -- [Coverage.py](https://coverage.readthedocs.io/en/coverage-5.0/cmd.html#xml-reporting) (Python) +- [Coverage.py](https://coverage.readthedocs.io/en/coverage-5.0.4/cmd.html#xml-reporting) (Python) Once configured, if you create a merge request that triggers a pipeline which collects coverage reports, the coverage will be shown in the diff view. This includes reports diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index 8897ac867d1..42b1570d213 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -7,9 +7,6 @@ description: "Automatic Let's Encrypt SSL certificates for GitLab Pages." > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) in GitLab 12.1. For versions earlier than GitLab 12.1, see the [manual Let's Encrypt instructions](../lets_encrypt_for_gitlab_pages.md). -NOTE: **Note:** -This feature is in **beta** and may still have bugs. See all the related issues linked from this [issue's description](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) for more information. - The GitLab Pages integration with Let's Encrypt (LE) allows you to use LE certificates for your Pages website with custom domains without the hassle of having to issue and update them yourself; @@ -63,18 +60,35 @@ associated Pages domain. It also will be renewed automatically by GitLab. ## Troubleshooting -### Error "Certificate misses intermediates" +### Error "Something went wrong while obtaining Let's Encrypt certificate" + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30146) in GitLab 13.0. -If you get an error **Certificate misses intermediates** while trying to enable Let's Encrypt integration for your domain, follow the steps below: +If you get an error **Something went wrong while obtaining Let's Encrypt certificate**, you can try obtaining the certificate again by following these steps: 1. Go to your project's **Settings > Pages**. -1. Turn off **Force HTTPS** if it's turned on. -1. Click **Details** on your domain. -1. Click the **Edit** button in the top right corner of domain details page. -1. Enable Let's Encrypt integration. -1. Click **Save**. +1. Click **Edit** on your domain. +1. Click **Retry**. +1. If you're still seeing the same error: + 1. Make sure you have properly set only one `CNAME` or `A` DNS record for your domain. + 1. Make sure your domain **doesn't have** an `AAAA` DNS record. + 1. If you have a `CAA` DNS record for your domain or any higher level domains, make sure [it includes `letsencrypt.org`](https://letsencrypt.org/docs/caa/). + 1. Make sure [your domain is verified](index.md#1-add-a-custom-domain-to-pages). + 1. Go to step 1. + +### Message "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later." hangs for more than an hour + +If you've enabled Let's Encrypt integration, but a certificate is absent after an hour and you see the message, "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later.", try to remove and add the domain for GitLab Pages again by following these steps: + 1. Go to your project's **Settings > Pages**. -1. Turn on **Force HTTPS**. +1. Click **Remove** on your domain. +1. [Add the domain again and verify it](index.md#1-add-a-custom-domain-to-pages). +1. [Enable Let's Encrypt integration for your domain](#enabling-lets-encrypt-integration-for-your-custom-domain). +1. If you still see the same message after some time: + 1. Make sure you have properly set only one `CNAME` or `A` DNS record for your domain. + 1. Make sure your domain **doesn't have** an `AAAA` DNS record. + 1. If you have a `CAA` DNS record for your domain or any higher level domains, make sure [it includes `letsencrypt.org`](https://letsencrypt.org/docs/caa/). + 1. Go to step 1. <!-- Include any troubleshooting steps that you can foresee. If you know beforehand what issues one might have when setting this up, or when something is changed, or on upgrading, it's diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index e121f7883be..f38cf8f139e 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -1,6 +1,6 @@ # Advanced Global Search **(STARTER ONLY)** -> - [Introduced][ee-109] in GitLab [Starter][ee] 8.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. > - This is the user documentation. To install and configure Elasticsearch, > visit the [administrator documentation](../../integration/elasticsearch.md). @@ -16,7 +16,7 @@ The Advanced Global Search in GitLab is a powerful search service that saves you time. Instead of creating duplicate code and wasting time, you can now search for code within other teams that can help your own project. -GitLab leverages the search capabilities of [Elasticsearch] and enables it when +GitLab leverages the search capabilities of [Elasticsearch](https://www.elastic.co/elasticsearch/) and enables it when searching in: - GitLab application @@ -56,20 +56,8 @@ project you have access to. You can also use the [Advanced Syntax Search](advanced_search_syntax.md) which provides some useful queries. ->**Note:** +NOTE: **Note:** Elasticsearch has only data for the default branch. That means that if you go to the repository tree and switch the branch from the default to something else, then the "Code" tab in the search result page will be served by the regular search even if Elasticsearch is enabled. - -[ee-1305]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1305 -[aws-elastic]: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html -[aws-iam]: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html -[aws-instance-profile]: http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli -[ee-109]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109 "Elasticsearch Merge Request" -[elasticsearch]: https://www.elastic.co/elasticsearch "Elasticsearch website" -[install]: https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html "Elasticsearch installation documentation" -[pkg]: https://about.gitlab.com/downloads/ "Download Omnibus GitLab" -[elastic-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html#settings "Elasticsearch configuration settings" -[ee]: https://about.gitlab.com/pricing/ -[es]: https://www.elastic.co/elasticsearch diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index faa3a118137..ebb957ad8e2 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -2,7 +2,7 @@ > **Notes:** > -> - Introduced in [GitLab Enterprise Starter][ee] 9.2 +> - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2 > - This is the user documentation. To install and configure Elasticsearch, > visit the [administrator documentation](../../integration/elasticsearch.md). @@ -37,7 +37,7 @@ better results. The Advanced Syntax Search supports fuzzy or exact search queries with prefixes, boolean operators, and much more. -Full details can be found in the [Elasticsearch documentation][elastic], but +Full details can be found in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/5.3/query-dsl-simple-query-string-query.html#_simple_query_string_syntax), but here's a quick guide: - Searches look for all the words in a query, in any order - e.g.: searching @@ -67,6 +67,3 @@ Examples: - Finding the text 'def create' inside files with the `.rb` extension: `def create extension:rb` - Finding the text `sha` inside files in a folder called `encryption`: `sha path:encryption` - Finding any file starting with `hello` containing `world` and with the `.js` extension: `world filename:hello* extension:js` - -[ee]: https://about.gitlab.com/pricing/ -[elastic]: https://www.elastic.co/guide/en/elasticsearch/reference/5.3/query-dsl-simple-query-string-query.html#_simple_query_string_syntax diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 407578fd4df..2166e8ddbd5 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -171,5 +171,3 @@ GitLab instance. Use advanced queries for more targeted search results. [Learn how to use the Advanced Syntax Search.](advanced_search_syntax.md) - -[ee]: https://about.gitlab.com/pricing/ diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index b2586714636..cd09d83b728 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -15,11 +15,6 @@ module Gitlab validations do validates :config, hash_or_string: true validates :config, allowed_keys: ALLOWED_KEYS - validate do - if config[:artifact] && config[:job].blank? - errors.add(:config, "must specify the job where to fetch the artifact from") - end - end end end end diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 020d1f323ee..10ef33e71d5 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings +# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables stages: @@ -19,26 +19,10 @@ dast: name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" variables: GIT_STRATEGY: none - # URL to scan: - # DAST_WEBSITE: https://example.com/ - # - # Time limit for target availability (scan is attempted even when timeout): - # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 - # - # Set these variables to scan with an authenticated user: - # DAST_AUTH_URL: https://example.com/sign-in - # DAST_USERNAME: john.doe@example.com - # DAST_PASSWORD: john-doe-password - # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form - # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form - # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between - # - # Perform ZAP Full Scan, which includes both passive and active scanning: - # DAST_FULL_SCAN_ENABLED: "true" allow_failure: true script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -t $DAST_WEBSITE + - /analyze artifacts: reports: dast: gl-dast-report.json diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 4b0062549f0..764047dae6d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -142,7 +142,6 @@ module Gitlab validate_job_stage!(name, job) validate_job_dependencies!(name, job) validate_job_needs!(name, job) - validate_dynamic_child_pipeline_dependencies!(name, job) validate_job_environment!(name, job) end end @@ -164,50 +163,35 @@ module Gitlab def validate_job_dependencies!(name, job) return unless job[:dependencies] - job[:dependencies].each do |dependency| - validate_job_dependency!(name, dependency) - end - end + stage_index = @stages.index(job[:stage]) - def validate_dynamic_child_pipeline_dependencies!(name, job) - return unless includes = job.dig(:trigger, :include) + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - includes.each do |included| - next unless dependency = included[:job] + dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage]) - validate_job_dependency!(name, dependency) + unless dependency_stage_index.present? && dependency_stage_index < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end end end def validate_job_needs!(name, job) - return unless needs = job.dig(:needs, :job) + return unless job.dig(:needs, :job) - needs.each do |need| - dependency = need[:name] - validate_job_dependency!(name, dependency, 'need') - end - end + stage_index = @stages.index(job[:stage]) - def validate_job_dependency!(name, dependency, dependency_type = 'dependency') - unless @jobs[dependency.to_sym] - raise ValidationError, "#{name} job: undefined #{dependency_type}: #{dependency}" - end + job.dig(:needs, :job).each do |need| + need_job_name = need[:name] - job_stage_index = stage_index(name) - dependency_stage_index = stage_index(dependency) + raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym] - # A dependency might be defined later in the configuration - # with a stage that does not exist - unless dependency_stage_index.present? && dependency_stage_index < job_stage_index - raise ValidationError, "#{name} job: #{dependency_type} #{dependency} is not defined in prior stages" - end - end + needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage]) - def stage_index(name) - job = @jobs[name.to_sym] - return unless job - - @stages.index(job[:stage]) + unless needs_stage_index.present? && needs_stage_index < stage_index + raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages" + end + end end def validate_job_environment!(name, job) diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb new file mode 100644 index 00000000000..e74fdd74049 --- /dev/null +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class NdjsonWriter + include Gitlab::ImportExport::CommandLineUtil + + def initialize(dir_path) + @dir_path = dir_path + end + + def close + end + + def write_attributes(exportable_path, hash) + # It will create: + # tree/project.json + with_file("#{exportable_path}.json") do |file| + file.write(hash.to_json) + end + end + + def write_relation(exportable_path, relation, value) + # It will create: + # tree/project/ci_cd_setting.ndjson + with_file(exportable_path, "#{relation}.ndjson") do |file| + file.write(value.to_json) + end + end + + def write_relation_array(exportable_path, relation, items) + # It will create: + # tree/project/merge_requests.ndjson + with_file(exportable_path, "#{relation}.ndjson") do |file| + items.each do |item| + file.write(item.to_json) + file.write("\n") + end + end + end + + private + + def with_file(*path) + file_path = File.join(@dir_path, *path) + raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path) + + # ensure that path is created + mkdir_p(File.dirname(file_path)) + + File.open(file_path, "wb") do |file| + yield(file) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 988776fe600..0017aa523c1 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -11,15 +11,9 @@ module Gitlab @project = project @current_user = current_user @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - json_writer = ImportExport::JSON::LegacyWriter.new( - @full_path, - allowed_path: "project" - ) - ImportExport::JSON::StreamingSerializer.new( exportable, reader.project_tree, @@ -57,6 +51,18 @@ module Gitlab def presenter_class Projects::ImportExport::ProjectExportPresenter end + + def json_writer + @json_writer ||= begin + if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) + else + full_path = File.join(@shared.export_path, ImportExport.project_filename) + Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project') + end + end + end end end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 37165d787c7..1b155570f18 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -29,12 +29,12 @@ module Gitlab # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)` def self.client_configurator lambda do |chain| - chain.add ::Gitlab::SidekiqStatus::ClientMiddleware - chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware chain.add ::Labkit::Middleware::Sidekiq::Client - chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client + chain.add ::Gitlab::SidekiqStatus::ClientMiddleware + chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client + chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics end end end diff --git a/spec/features/issues/filtered_search/filter_issues_spec.rb b/spec/features/issues/filtered_search/filter_issues_spec.rb index 756699fb854..5e2b5921e06 100644 --- a/spec/features/issues/filtered_search/filter_issues_spec.rb +++ b/spec/features/issues/filtered_search/filter_issues_spec.rb @@ -224,14 +224,6 @@ describe 'Filter issues', :js do expect_no_issues_list expect_filtered_search_input_empty end - - it 'does show issues for bug label' do - input_filtered_search("label:!=~#{bug_label.title}") - - expect_tokens([label_token(bug_label.title)]) - expect_issues_list_count(6) - expect_filtered_search_input_empty - end end context 'label with multiple words' do diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index 54a6ac1551b..1e13afc6033 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -38,51 +38,109 @@ describe 'Import/Export - project export integration test', :js do sign_in(user) end - it 'exports a project successfully', :sidekiq_might_not_need_inline do - visit edit_project_path(project) + shared_examples 'export file without sensitive words' do + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) - expect(page).to have_content('Export project') + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) - find(:link, 'Export project').send_keys(:return) + project_json_path = File.join(tmpdir, 'project.json') + expect(File).to exist(project_json_path) - visit edit_project_path(project) + project_hash = JSON.parse(IO.read(project_json_path)) - expect(page).to have_content('Download export') + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, project_hash) - expect(project.export_status).to eq(:finished) - expect(project.export_file.path).to include('tar.gz') + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end + end + end + + context "with legacy export" do + before do + stub_feature_flags(streaming_serializer: false) + stub_feature_flags(project_export_as_ndjson: false) + end + + it_behaves_like "export file without sensitive words" + end + + context "with streaming serializer" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: false) + end + + it_behaves_like "export file without sensitive words" + end - in_directory_with_expanded_export(project) do |exit_status, tmpdir| - expect(exit_status).to eq(0) + context "with ndjson" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: true) + end + + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) + + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) - project_json_path = File.join(tmpdir, 'project.json') - expect(File).to exist(project_json_path) + project_json_path = File.join(tmpdir, 'tree', 'project.json') + expect(File).to exist(project_json_path) - project_hash = JSON.parse(IO.read(project_json_path)) + relations = [] + relations << JSON.parse(IO.read(project_json_path)) + Dir.glob(File.join(tmpdir, 'tree/project', '*.ndjson')) do |rb_filename| + File.foreach(rb_filename) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + end - sensitive_words.each do |sensitive_word| - found = find_sensitive_attributes(sensitive_word, project_hash) + relations.each do |relation_hash| + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, relation_hash) - expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end end end end + end - def failure_message(key_found, parent, sensitive_word) - <<-MSG - Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + def export_project_and_download_file(page, project) + visit edit_project_path(project) - If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + expect(page).to have_content('Export project') - Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the - correspondent hash or model as the value. + find(:link, 'Export project').send_keys(:return) - Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be - reset (to prevent duplicate column problems while importing to the same instance). + visit edit_project_path(project) - IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} - CURRENT_SPEC: #{__FILE__} - MSG - end + expect(page).to have_content('Download export') + expect(project.export_status).to eq(:finished) + expect(project.export_file.path).to include('tar.gz') + end + + def failure_message(key_found, parent, sensitive_word) + <<-MSG + Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + + If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + + Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the + correspondent hash or model as the value. + + Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be + reset (to prevent duplicate column problems while importing to the same instance). + + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + CURRENT_SPEC: #{__FILE__} + MSG end end diff --git a/spec/finders/environments_finder_spec.rb b/spec/finders/environments_finder_spec.rb index 7100376478a..850d160506c 100644 --- a/spec/finders/environments_finder_spec.rb +++ b/spec/finders/environments_finder_spec.rb @@ -37,11 +37,6 @@ describe EnvironmentsFinder do .to be_empty end - it 'returns environment when with_tags is set' do - expect(described_class.new(project, user, ref: 'master', commit: commit, with_tags: true).execute) - .to contain_exactly(environment, environment_two) - end - # We expect two Gitaly calls: FindCommit, CommitIsAncestor # This tests to ensure we don't call one CommitIsAncestor per environment it 'only calls Gitaly twice when multiple environments are present', :request_store do diff --git a/spec/helpers/users_helper_spec.rb b/spec/helpers/users_helper_spec.rb index 893d5cde24a..e3e599007a4 100644 --- a/spec/helpers/users_helper_spec.rb +++ b/spec/helpers/users_helper_spec.rb @@ -95,9 +95,9 @@ describe UsersHelper do end it 'includes the settings tab if the user can update themself' do - expect(helper).to receive(:can?).with(user, :read_user, user) { true } + expect(helper).to receive(:can?).with(user, :update_user, user) { true } - expect(items).to include(:profile) + expect(items).to include(:settings) end context 'when terms are enforced' do diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 0b34e887716..62adba4319e 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1647,48 +1647,6 @@ module Gitlab it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) } end - - context 'when trigger job includes artifact generated by a dependency' do - context 'when dependency is defined in previous stages' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'build1', artifact: 'generated.yml' }] - } } - } - end - - it { expect { subject }.not_to raise_error } - end - - context 'when dependency is defined in later stages' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'deploy1', artifact: 'generated.yml' }] - } }, - deploy1: { stage: 'deploy', script: 'test' } - } - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) } - end - - context 'when dependency is not defined' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'non-existent', artifact: 'generated.yml' }] - } } - } - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /undefined dependency: non-existent/) } - end - end end describe "Job Needs" do @@ -2094,34 +2052,6 @@ module Gitlab end end - describe 'with trigger:include' do - context 'when artifact and job are specified' do - let(:config) do - YAML.dump({ - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ artifact: 'generated.yml', job: 'build1' }] - } } - }) - end - - it { expect { subject }.not_to raise_error } - end - - context 'when artifact is specified without job' do - let(:config) do - YAML.dump({ - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ artifact: 'generated.yml' }] - } } - }) - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /must specify the job where to fetch the artifact from/) } - end - end - describe "Error handling" do it "fails to parse YAML" do expect do diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb index 8aa28353c04..bb79331efac 100644 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ b/spec/lib/gitlab/import_export/fork_spec.rb @@ -32,6 +32,8 @@ describe 'forked project import' do end before do + stub_feature_flags(project_export_as_ndjson: false) + allow_next_instance_of(Gitlab::ImportExport) do |instance| allow(instance).to receive(:storage_path).and_return(export_path) end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb index 50b26637cb1..707975f20b6 100644 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb @@ -20,6 +20,10 @@ describe Gitlab::ImportExport do let(:json_fixture) { 'complex' } + before do + stub_feature_flags(project_export_as_ndjson: false) + end + it 'yields the initial tree when importing and exporting it again' do project = create(:project, creator: create(:user, :admin)) diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb new file mode 100644 index 00000000000..bae3672474c --- /dev/null +++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Gitlab::ImportExport::JSON::NdjsonWriter do + include ImportExport::CommonUtil + + let(:path) { "#{Dir.tmpdir}/ndjson_writer_spec/tree" } + let(:exportable_path) { 'projects' } + + subject { described_class.new(path) } + + after do + FileUtils.rm_rf(path) + end + + describe "#write_attributes" do + it "writes correct json to root" do + expected_hash = { "key" => "value_1", "key_1" => "value_2" } + subject.write_attributes(exportable_path, expected_hash) + + expect(consume_attributes(path, exportable_path)).to eq(expected_hash) + end + end + + describe "#write_relation" do + context "when single relation is serialized" do + it "appends json in correct file " do + relation = "relation" + value = { "key" => "value_1", "key_1" => "value_1" } + subject.write_relation(exportable_path, relation, value) + + expect(consume_relations(path, exportable_path, relation)).to eq([value]) + end + end + + context "when single relation is already serialized" do + it "raise exception" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relation = "relation" + file_path = File.join(path, exportable_path, "#{relation}.ndjson") + subject.write_relation(exportable_path, relation, values[0]) + + expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist") + end + end + end + + describe "#write_relation_array" do + it "writes json in correct files" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relations = %w(relation1 relation2) + relations.each do |relation| + subject.write_relation_array(exportable_path, relation, values.to_enum) + end + subject.close + + relations.each do |relation| + expect(consume_relations(path, exportable_path, relation)).to eq(values) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb index 73ae6810706..175da623c1b 100644 --- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb @@ -160,7 +160,7 @@ describe Gitlab::ImportExport::Project::RelationFactory do end it 'has preloaded target project' do - expect(created_object.source_project).to equal(project) + expect(created_object.target_project).to equal(project) end end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index c910ee5430b..d859af5df02 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -3,314 +3,397 @@ require 'spec_helper' describe Gitlab::ImportExport::Project::TreeSaver do - describe 'saves the project tree into a json object' do - let(:shared) { project.import_export_shared } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:user) { create(:user) } - let!(:project) { setup_project } - - before do - project.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') - allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') - end + let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let_it_be(:exportable_path) { 'project' } - after do - FileUtils.rm_rf(export_path) - end + shared_examples 'saves project tree successfully' do |ndjson_enabled| + include ImportExport::CommonUtil - it 'saves project successfully' do - expect(project_tree_saver.save).to be true - end + subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } - context 'JSON' do - let(:saved_project_json) do - project_tree_saver.save - project_json(project_tree_saver.full_path) - end + describe 'saves project tree attributes' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { setup_project } + let_it_be(:shared) { project.import_export_shared } + let_it_be(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared) } - # It is not duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - context 'with description override' do - let(:params) { { description: 'Foo Bar' } } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + let(:relation_name) { :projects } - it 'overrides the project description' do - expect(saved_project_json).to include({ 'description' => params[:description] }) + let_it_be(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) end end - it 'saves the correct json' do - expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) + before_all do + Feature.enable(:project_export_as_ndjson) if ndjson_enabled + project.add_maintainer(user) + project_tree_saver.save end - it 'has approvals_before_merge set' do - expect(saved_project_json['approvals_before_merge']).to eq(1) + after :all do + FileUtils.rm_rf(export_path) end - it 'has milestones' do - expect(saved_project_json['milestones']).not_to be_empty - end + context 'with project root' do + it { is_expected.to include({ 'description' => 'description', 'visibility_level' => 20 }) } - it 'has merge requests' do - expect(saved_project_json['merge_requests']).not_to be_empty - end + it { is_expected.not_to include("runners_token" => 'token') } - it 'has merge request\'s milestones' do - expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty + it 'has approvals_before_merge set' do + expect(subject['approvals_before_merge']).to eq(1) + end end - it 'has merge request\'s source branch SHA' do - expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD') - end + context 'with milestones' do + let(:relation_name) { :milestones } - it 'has merge request\'s target branch SHA' do - expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA') + it { is_expected.not_to be_empty } end - it 'has events' do - expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty - end + context 'with merge_requests' do + let(:relation_name) { :merge_requests } - it 'has snippets' do - expect(saved_project_json['snippets']).not_to be_empty - end + it { is_expected.not_to be_empty } - it 'has snippet notes' do - expect(saved_project_json['snippets'].first['notes']).not_to be_empty - end + it 'has merge request\'s milestones' do + expect(subject.first['milestone']).not_to be_empty + end + it 'has merge request\'s source branch SHA' do + expect(subject.first['source_branch_sha']).to eq('b83d6e391c22777fca1ed3012fce84f633d7fed0') + end - it 'has releases' do - expect(saved_project_json['releases']).not_to be_empty - end + it 'has merge request\'s target branch SHA' do + expect(subject.first['target_branch_sha']).to eq('0b4bc9a49b562e85de7cc9e834518ea6828729b9') + end - it 'has no author on releases' do - expect(saved_project_json['releases'].first['author']).to be_nil - end + it 'has events' do + expect(subject.first['milestone']['events']).not_to be_empty + end - it 'has the author ID on releases' do - expect(saved_project_json['releases'].first['author_id']).not_to be_nil - end + it 'has merge requests diffs' do + expect(subject.first['merge_request_diff']).not_to be_empty + end - it 'has issues' do - expect(saved_project_json['issues']).not_to be_empty - end + it 'has merge request diff files' do + expect(subject.first['merge_request_diff']['merge_request_diff_files']).not_to be_empty + end - it 'has issue comments' do - notes = saved_project_json['issues'].first['notes'] + it 'has merge request diff commits' do + expect(subject.first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty + end - expect(notes).not_to be_empty - expect(notes.first['type']).to eq('DiscussionNote') - end + it 'has merge requests comments' do + expect(subject.first['notes']).not_to be_empty + end - it 'has issue assignees' do - expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty - end + it 'has author on merge requests comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end - it 'has author on issue comments' do - expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty + it 'has merge request resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end end - it 'has system note metadata on issue comments' do - metadata = saved_project_json['issues'].first['notes'].first['system_note_metadata'] + context 'with snippets' do + let(:relation_name) { :snippets } - expect(metadata['action']).to eq('description') - end + it { is_expected.not_to be_empty } - it 'has project members' do - expect(saved_project_json['project_members']).not_to be_empty + it 'has snippet notes' do + expect(subject.first['notes']).not_to be_empty + end end - it 'has merge requests diffs' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty - end + context 'with releases' do + let(:relation_name) { :releases } - it 'has merge request diff files' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty - end + it { is_expected.not_to be_empty } - it 'has merge request diff commits' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty - end + it 'has no author on releases' do + expect(subject.first['author']).to be_nil + end - it 'has merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty + it 'has the author ID on releases' do + expect(subject.first['author_id']).not_to be_nil + end end - it 'has author on merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty - end + context 'with issues' do + let(:relation_name) { :issues } - it 'has system note metadata on merge requests comments' do - metadata = saved_project_json['merge_requests'].first['notes'].first['system_note_metadata'] + it { is_expected.not_to be_empty } - expect(metadata['commit_count']).to eq(1) - expect(metadata['action']).to eq('commit') - end + it 'has issue comments' do + notes = subject.first['notes'] - it 'has pipeline stages' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty - end + expect(notes).not_to be_empty + expect(notes.first['type']).to eq('DiscussionNote') + end - it 'has pipeline statuses' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty - end + it 'has issue assignees' do + expect(subject.first['issue_assignees']).not_to be_empty + end - it 'has pipeline builds' do - builds_count = saved_project_json - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } + it 'has author on issue comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end - expect(builds_count).to eq(1) - end + it 'has labels associated to records' do + expect(subject.first['label_links'].first['label']).not_to be_empty + end - it 'has no when YML attributes but only the DB column' do - expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + it 'has project and group labels' do + label_types = subject.first['label_links'].map { |link| link['label']['type'] } - saved_project_json - end + expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + end - it 'has pipeline commits' do - expect(saved_project_json['ci_pipelines']).not_to be_empty - end + it 'has priorities associated to labels' do + priorities = subject.first['label_links'].flat_map { |link| link['label']['priorities'] } - it 'has ci pipeline notes' do - expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty - end + expect(priorities).not_to be_empty + end - it 'has labels with no associations' do - expect(saved_project_json['labels']).not_to be_empty + it 'has issue resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end end - it 'has labels associated to records' do - expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty - end + context 'with ci_pipelines' do + let(:relation_name) { :ci_pipelines } + + it { is_expected.not_to be_empty } + + it 'has pipeline stages' do + expect(subject.dig(0, 'stages')).not_to be_empty + end + + it 'has pipeline statuses' do + expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + end - it 'has project and group labels' do - label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] } + it 'has pipeline builds' do + builds_count = subject.dig(0, 'stages', 0, 'statuses') + .count { |hash| hash['type'] == 'Ci::Build' } - expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + expect(builds_count).to eq(1) + end + + it 'has ci pipeline notes' do + expect(subject.first['notes']).not_to be_empty + end end - it 'has priorities associated to labels' do - priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] } + context 'with labels' do + let(:relation_name) { :labels } - expect(priorities).not_to be_empty + it { is_expected.not_to be_empty } end - it 'has issue resource label events' do - expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty + context 'with services' do + let(:relation_name) { :services } + + it 'saves the correct service type' do + expect(subject.first['type']).to eq('CustomIssueTrackerService') + end + + it 'saves the properties for a service' do + expect(subject.first['properties']).to eq('one' => 'value') + end end - it 'has merge request resource label events' do - expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty + context 'with project_feature' do + let(:relation_name) { :project_feature } + + it { is_expected.not_to be_empty } + + it 'has project feature' do + expect(subject["issues_access_level"]).to eq(ProjectFeature::DISABLED) + expect(subject["wiki_access_level"]).to eq(ProjectFeature::ENABLED) + expect(subject["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + end end - it 'saves the correct service type' do - expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService') + context 'with custom_attributes' do + let(:relation_name) { :custom_attributes } + + it 'has custom attributes' do + expect(subject.count).to eq(2) + end end - it 'saves the properties for a service' do - expect(saved_project_json['services'].first['properties']).to eq('one' => 'value') + context 'with badges' do + let(:relation_name) { :custom_attributes } + + it 'has badges' do + expect(subject.count).to eq(2) + end end - it 'has project feature' do - project_feature = saved_project_json['project_feature'] - expect(project_feature).not_to be_empty - expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) - expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) - expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + context 'with project_members' do + let(:relation_name) { :project_members } + + it { is_expected.not_to be_empty } end - it 'has custom attributes' do - expect(saved_project_json['custom_attributes'].count).to eq(2) + context 'with boards' do + let(:relation_name) { :boards } + + it { is_expected.not_to be_empty } end + end - it 'has badges' do - expect(saved_project_json['project_badges'].count).to eq(2) + describe '#saves project tree' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let(:project) { setup_project } + let(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end end + let(:shared) { project.import_export_shared } + let(:params) { {} } - it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } - expect(project_tree_saver.save).to be true + before do + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + project.add_maintainer(user) + + FileUtils.rm_rf(export_path) end - context 'group members' do + after do + FileUtils.rm_rf(export_path) + end + + context 'overrides group members' do let(:user2) { create(:user, email: 'group@member.com') } + let(:relation_name) { :project_members } + let(:member_emails) do - saved_project_json['project_members'].map do |pm| + emails = subject.map do |pm| pm['user']['email'] end + emails end before do - Group.first.add_developer(user2) + group.add_developer(user2) end - it 'does not export group members if it has no permission' do - Group.first.add_developer(user) + context 'when has no permission' do + before do + group.add_developer(user) + project_tree_saver.save + end - expect(member_emails).not_to include('group@member.com') + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end end - it 'does not export group members as maintainer' do - Group.first.add_maintainer(user) + context 'when has permission as maintainer' do + before do + group.add_maintainer(user) - expect(member_emails).not_to include('group@member.com') + project_tree_saver.save + end + + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end end - it 'exports group members as group owner' do - Group.first.add_owner(user) + context 'when has permission as group owner' do + before do + group.add_owner(user) - expect(member_emails).to include('group@member.com') + project_tree_saver.save + end + + it 'exports group members as group owner' do + expect(member_emails).to include('group@member.com') + end end context 'as admin' do let(:user) { create(:admin) } + before do + project_tree_saver.save + end + it 'exports group members as admin' do expect(member_emails).to include('group@member.com') end it 'exports group members as project members' do - member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] } + member_types = subject.map { |pm| pm['source_type'] } expect(member_types).to all(eq('Project')) end end end - context 'project attributes' do - it 'does not contain the runners token' do - expect(saved_project_json).not_to include("runners_token" => 'token') + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:relation_name) { :projects } + + before do + project_tree_saver.save end + + it { is_expected.to include({ 'description' => params[:description] }) } + end + + it 'saves project successfully' do + expect(project_tree_saver.save).to be true + end + + it 'does not complain about non UTF-8 characters in MR diff files' do + ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + + expect(project_tree_saver.save).to be true end - it 'has a board and a list' do - expect(saved_project_json['boards'].first['lists']).not_to be_empty + it 'has no when YML attributes but only the DB column' do + expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + + project_tree_saver.save end end end + context 'with JSON' do + it_behaves_like "saves project tree successfully", false + end + + context 'with NDJSON' do + it_behaves_like "saves project tree successfully", true + end + def setup_project release = create(:release) - group = create(:group) project = create(:project, - :public, - :repository, - :issues_disabled, - :wiki_enabled, - :builds_private, - description: 'description', - releases: [release], - group: group, - approvals_before_merge: 1 - ) - allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project)) + :public, + :repository, + :issues_disabled, + :wiki_enabled, + :builds_private, + description: 'description', + releases: [release], + group: group, + approvals_before_merge: 1) issue = create(:issue, assignees: [user], project: project) snippet = create(:project_snippet, project: project) @@ -331,9 +414,9 @@ describe Gitlab::ImportExport::Project::TreeSaver do mr_note = create(:note, noteable: merge_request, project: project) create(:note, noteable: snippet, project: project) create(:note_on_commit, - author: user, - project: project, - commit_id: ci_build.pipeline.sha) + author: user, + project: project, + commit_id: ci_build.pipeline.sha) create(:system_note_metadata, action: 'description', note: discussion_note) create(:system_note_metadata, commit_count: 1, action: 'commit', note: mr_note) @@ -355,8 +438,4 @@ describe Gitlab::ImportExport::Project::TreeSaver do project end - - def project_json(filename) - ::JSON.parse(IO.read(filename)) - end end diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index 88f83ebc2ac..32c1807ba6e 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -134,12 +134,12 @@ describe Gitlab::SidekiqMiddleware do let(:middleware_expected_args) { [worker_class_arg, job, queue, redis_pool] } let(:expected_middlewares) do [ - Gitlab::SidekiqStatus::ClientMiddleware, - Gitlab::SidekiqMiddleware::ClientMetrics, - Gitlab::SidekiqMiddleware::WorkerContext::Client, - Labkit::Middleware::Sidekiq::Client, - Gitlab::SidekiqMiddleware::AdminMode::Client, - Gitlab::SidekiqMiddleware::DuplicateJobs::Client + ::Gitlab::SidekiqMiddleware::WorkerContext::Client, + ::Labkit::Middleware::Sidekiq::Client, + ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client, + ::Gitlab::SidekiqStatus::ClientMiddleware, + ::Gitlab::SidekiqMiddleware::AdminMode::Client, + ::Gitlab::SidekiqMiddleware::ClientMetrics ] end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 137795dcbc3..bfad1da1e21 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -3701,4 +3701,18 @@ describe MergeRequest do end end end + + describe '#predefined_variables' do + let(:merge_request) { create(:merge_request) } + + it 'caches all SQL-sourced data on the first call' do + control = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count + + expect(control).to be > 0 + + count = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count + + expect(count).to eq(0) + end + end end diff --git a/spec/services/projects/hashed_storage/migration_service_spec.rb b/spec/services/projects/hashed_storage/migration_service_spec.rb index f3ac26e7761..0a7975305dc 100644 --- a/spec/services/projects/hashed_storage/migration_service_spec.rb +++ b/spec/services/projects/hashed_storage/migration_service_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' describe Projects::HashedStorage::MigrationService do let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) } let(:logger) { double } + let!(:project_attachment) { build(:file_uploader, project: project) } + let(:project_hashed_path) { Storage::Hashed.new(project).disk_path } + let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path } + let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" } + let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" } subject(:service) { described_class.new(project, project.full_path, logger: logger) } @@ -29,9 +34,24 @@ describe Projects::HashedStorage::MigrationService do service.execute end + + it 'migrates legacy repositories to hashed storage' do + legacy_attachments_path = FileUploader.absolute_base_dir(project) + hashed_project = project.dup.tap { |p| p.id = project.id } + hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project) + + expect(logger).to receive(:info).with(/Repository moved from '#{project_legacy_path}' to '#{project_hashed_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_legacy_path}' to '#{wiki_hashed_path}'/) + expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(nil).to(2) + end end context 'attachments migration' do + let(:project) { create(:project, :empty_repo, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) } + let(:attachments_service) do Projects::HashedStorage::MigrateAttachmentsService.new(project: project, old_disk_path: project.full_path, @@ -51,6 +71,17 @@ describe Projects::HashedStorage::MigrationService do service.execute end + + it 'migrates legacy attachments to hashed storage' do + legacy_attachments_path = FileUploader.absolute_base_dir(project) + hashed_project = project.dup.tap { |p| p.id = project.id } + hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project) + + expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(1).to(2) + end end end end diff --git a/spec/services/projects/hashed_storage/rollback_service_spec.rb b/spec/services/projects/hashed_storage/rollback_service_spec.rb index 48d4eac9eb7..e6b7daba99e 100644 --- a/spec/services/projects/hashed_storage/rollback_service_spec.rb +++ b/spec/services/projects/hashed_storage/rollback_service_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' describe Projects::HashedStorage::RollbackService do let(:project) { create(:project, :empty_repo, :wiki_repo) } let(:logger) { double } + let!(:project_attachment) { build(:file_uploader, project: project) } + let(:project_hashed_path) { Storage::Hashed.new(project).disk_path } + let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path } + let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" } + let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" } subject(:service) { described_class.new(project, project.disk_path, logger: logger) } @@ -26,6 +31,20 @@ describe Projects::HashedStorage::RollbackService do service.execute end + + it 'rollbacks to legacy storage' do + hashed_attachments_path = FileUploader.absolute_base_dir(project) + legacy_project = project.dup + legacy_project.storage_version = nil + legacy_attachments_path = FileUploader.absolute_base_dir(legacy_project) + + expect(logger).to receive(:info).with(/Project attachments moved from '#{hashed_attachments_path}' to '#{legacy_attachments_path}'/) + + expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(2).to(nil) + end end context 'repository rollback' do @@ -47,6 +66,13 @@ describe Projects::HashedStorage::RollbackService do service.execute end + + it 'rollbacks to legacy storage' do + expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(1).to(nil) + end end end end diff --git a/spec/services/web_hook_service_spec.rb b/spec/services/web_hook_service_spec.rb index d8f13bc2e61..4a917ecdbb5 100644 --- a/spec/services/web_hook_service_spec.rb +++ b/spec/services/web_hook_service_spec.rb @@ -71,16 +71,6 @@ describe WebHookService do end end - it 'POSTs to the webhook URL' do - stub_full_request(project_hook.url, method: :post) - - service_instance.execute - - expect(WebMock).to have_requested(:post, stubbed_hostname(project_hook.url)).with( - headers: headers - ).once - end - it 'POSTs the data as JSON' do stub_full_request(project_hook.url, method: :post) diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb index 9281937e4ba..efe14b7244c 100644 --- a/spec/support/import_export/common_util.rb +++ b/spec/support/import_export/common_util.rb @@ -26,6 +26,21 @@ module ImportExport "tmp/tests/gitlab-test/import_export" end + def get_json(path, exportable_path, key, ndjson_enabled) + if ndjson_enabled + json = if key == :projects + consume_attributes(path, exportable_path) + else + consume_relations(path, exportable_path, key) + end + else + json = project_json(path) + json = json[key.to_s] unless key == :projects + end + + json + end + def restore_then_save_project(project, import_path:, export_path:) project_restorer = get_project_restorer(project, import_path) project_saver = get_project_saver(project, export_path) @@ -50,5 +65,30 @@ module ImportExport allow(shared).to receive(:export_path).and_return(path) end end + + def consume_attributes(dir_path, exportable_path) + path = File.join(dir_path, "#{exportable_path}.json") + return unless File.exist?(path) + + ActiveSupport::JSON.decode(IO.read(path)) + end + + def consume_relations(dir_path, exportable_path, key) + path = File.join(dir_path, exportable_path, "#{key}.ndjson") + return unless File.exist?(path) + + relations = [] + + File.foreach(path) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + + key == :project_feature ? relations.first : relations.flatten + end + + def project_json(filename) + ActiveSupport::JSON.decode(IO.read(filename)) + end end end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb index 5f9316c1cde..5f9316c1cde 100644 --- a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb |