diff options
57 files changed, 1030 insertions, 144 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..d704f20c726 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{js,json,vue,scss,rb,haml,yml}] +indent_size = 2 + +[*.{js,json,vue,scss,rb,haml,yml,md}] +indent_style = space +charset = utf-8 diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 06b32856e0e..28ad2b41921 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -56,6 +56,7 @@ Dangerfile @gl-quality/eng-prod /lib/gitlab/danger/ @gl-quality/eng-prod /scripts/ @gl-quality/eng-prod /scripts/frontend/ @gl-quality/eng-prod @gitlab-org/maintainers/frontend +.editorconfig @gl-quality/eng-prod # Telemetry owner files /ee/lib/gitlab/usage_data_counters/ @gitlab-org/growth/telemetry diff --git a/.gitlab/issue_templates/Security Release.md b/.gitlab/issue_templates/Security Release.md index a3689bff450..b06f31f0e9a 100644 --- a/.gitlab/issue_templates/Security Release.md +++ b/.gitlab/issue_templates/Security Release.md @@ -4,6 +4,10 @@ Set the title to: `Security Release: 12.2.X, 12.1.X, and 12.0.X` --> +:warning: **Only Release Managers and members of the AppSec team can edit the description of this issue** + +------- + ## Releases tasks - https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/release-manager.md @@ -24,22 +28,6 @@ in the "Linked issues" section below this issue description. tracking issue, their merge requests may not be included in the security release. -## Issues in Omnibus-GitLab - -Omnibus security fixes need to be added manually to this issue description -using and below the following template: - -```markdown -* {https://gitlab.com/gitlab-org/security/gitlab/issues/ link} - -| Version | MR | -|---------|----| -| 12.2 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | -| 12.1 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | -| 12.0 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | -| master | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | -``` - ## QA {QA issue link} diff --git a/CHANGELOG-EE.md b/CHANGELOG-EE.md index 0144ebde626..fe44d526d51 100644 --- a/CHANGELOG-EE.md +++ b/CHANGELOG-EE.md @@ -1,5 +1,12 @@ Please view this file on the master branch, on stable branches it's out of date. +## 12.8.7 (2020-03-16) + +### Fixed (1 change) + +- Allow multipart uploads for packages. !26387 + + ## 12.8.6 (2020-03-11) - No changes. diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fb12b3a88..a75434d1ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ entry. ## 12.8.7 (2020-03-16) -- No changes. +### Fixed (1 change, 1 of them is from the community) + +- Fix crl_url parsing and certificate visualization. !25876 (Roger Meier) + ## 12.8.6 (2020-03-11) diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue index 59c5586edcd..aefb31fe3d5 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue +++ b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue @@ -1,6 +1,6 @@ <script> import { createNamespacedHelpers, mapState, mapActions } from 'vuex'; -import _ from 'underscore'; +import { escape as esc } from 'lodash'; import { GlFormInput, GlFormCheckbox } from '@gitlab/ui'; import { sprintf, s__ } from '~/locale'; import ClusterFormDropdown from '~/create_cluster/components/cluster_form_dropdown.vue'; @@ -133,7 +133,7 @@ export default { : s__('ClusterIntegration|Create Kubernetes cluster'); }, kubernetesIntegrationHelpText() { - const escapedUrl = _.escape(this.kubernetesIntegrationHelpPath); + const escapedUrl = esc(this.kubernetesIntegrationHelpPath); return sprintf( s__( @@ -245,7 +245,7 @@ export default { ); }, gitlabManagedHelpText() { - const escapedUrl = _.escape(this.gitlabManagedClusterHelpPath); + const escapedUrl = esc(this.gitlabManagedClusterHelpPath); return sprintf( s__( diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/service_credentials_form.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/service_credentials_form.vue index 0cfe47dafaf..47cc4e4ce67 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/service_credentials_form.vue +++ b/app/assets/javascripts/create_cluster/eks_cluster/components/service_credentials_form.vue @@ -1,6 +1,6 @@ <script> import { GlFormInput } from '@gitlab/ui'; -import _ from 'underscore'; +import { escape as esc } from 'lodash'; import { mapState, mapActions } from 'vuex'; import { sprintf, s__, __ } from '~/locale'; import ClipboardButton from '~/vue_shared/components/clipboard_button.vue'; @@ -42,7 +42,7 @@ export default { : s__('ClusterIntegration|Authenticate with AWS'); }, accountAndExternalIdsHelpText() { - const escapedUrl = _.escape(this.accountAndExternalIdsHelpPath); + const escapedUrl = esc(this.accountAndExternalIdsHelpPath); return sprintf( s__( @@ -59,7 +59,7 @@ export default { ); }, provisionRoleArnHelpText() { - const escapedUrl = _.escape(this.createRoleArnHelpPath); + const escapedUrl = esc(this.createRoleArnHelpPath); return sprintf( s__( diff --git a/app/assets/javascripts/create_cluster/gke_cluster/components/gke_dropdown_mixin.js b/app/assets/javascripts/create_cluster/gke_cluster/components/gke_dropdown_mixin.js index 43fd0cac3be..3b91ce63744 100644 --- a/app/assets/javascripts/create_cluster/gke_cluster/components/gke_dropdown_mixin.js +++ b/app/assets/javascripts/create_cluster/gke_cluster/components/gke_dropdown_mixin.js @@ -1,4 +1,3 @@ -import _ from 'underscore'; import { GlLoadingIcon } from '@gitlab/ui'; import DropdownSearchInput from '~/vue_shared/components/dropdown/dropdown_search_input.vue'; import DropdownHiddenInput from '~/vue_shared/components/dropdown/dropdown_hidden_input.vue'; @@ -49,7 +48,7 @@ export default { methods: { fetchSuccessHandler() { if (this.defaultValue) { - const itemToSelect = _.find(this.items, item => item.name === this.defaultValue); + const itemToSelect = this.items.find(item => item.name === this.defaultValue); if (itemToSelect) { this.setItem(itemToSelect.name); diff --git a/app/assets/javascripts/create_cluster/gke_cluster/components/gke_project_id_dropdown.vue b/app/assets/javascripts/create_cluster/gke_cluster/components/gke_project_id_dropdown.vue index 6815d3629e3..6d8e6bbac11 100644 --- a/app/assets/javascripts/create_cluster/gke_cluster/components/gke_project_id_dropdown.vue +++ b/app/assets/javascripts/create_cluster/gke_cluster/components/gke_project_id_dropdown.vue @@ -1,5 +1,5 @@ <script> -import _ from 'underscore'; +import { escape as esc } from 'lodash'; import { mapState, mapGetters, mapActions } from 'vuex'; import { s__, sprintf } from '~/locale'; @@ -65,7 +65,7 @@ export default { s__(message), { docsLinkEnd: ' <i class="fa fa-external-link" aria-hidden="true"></i></a>', - docsLinkStart: `<a href="${_.escape( + docsLinkStart: `<a href="${esc( this.docsUrl, )}" target="_blank" rel="noopener noreferrer">`, }, @@ -119,7 +119,7 @@ export default { ...mapActions({ setItem: 'setProject' }), fetchSuccessHandler() { if (this.defaultValue) { - const projectToSelect = _.find(this.items, item => item.projectId === this.defaultValue); + const projectToSelect = this.items.find(item => item.projectId === this.defaultValue); if (projectToSelect) { this.setItem(projectToSelect); diff --git a/app/helpers/submodule_helper.rb b/app/helpers/submodule_helper.rb index 1149b168383..e9554300075 100644 --- a/app/helpers/submodule_helper.rb +++ b/app/helpers/submodule_helper.rb @@ -38,6 +38,8 @@ module SubmoduleHelper url_helpers.namespace_project_tree_path(namespace, project, submodule_item_id)] elsif relative_self_url?(url) relative_self_links(url, submodule_item_id, repository.project) + elsif gist_github_dot_com_url?(url) + gist_github_com_tree_links(namespace, project, submodule_item_id) elsif github_dot_com_url?(url) github_com_tree_links(namespace, project, submodule_item_id) elsif gitlab_dot_com_url?(url) @@ -52,6 +54,10 @@ module SubmoduleHelper protected + def gist_github_dot_com_url?(url) + url =~ %r{gist\.github\.com[/:][^/]+/[^/]+\Z} + end + def github_dot_com_url?(url) url =~ %r{github\.com[/:][^/]+/[^/]+\Z} end @@ -78,6 +84,11 @@ module SubmoduleHelper [base, [base, '/-/tree/', commit].join('')] end + def gist_github_com_tree_links(namespace, project, commit) + base = ['https://gist.github.com/', namespace, '/', project].join('') + [base, [base, commit].join('/')] + end + def github_com_tree_links(namespace, project, commit) base = ['https://github.com/', namespace, '/', project].join('') [base, [base, '/tree/', commit].join('')] diff --git a/app/models/ability.rb b/app/models/ability.rb index 671a92632d5..514e923c380 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -44,7 +44,7 @@ class Ability # Returns an Array of MergeRequests that can be read by the given user. # - # merge_requests - MRs out of which to collect mr's readable by the user. + # merge_requests - MRs out of which to collect MRs readable by the user. # user - The User for which to check the merge_requests # filters - A hash of abilities and filters to apply if the user lacks this # ability diff --git a/app/services/commits/cherry_pick_service.rb b/app/services/commits/cherry_pick_service.rb index 91a18909e22..7e982bf7686 100644 --- a/app/services/commits/cherry_pick_service.rb +++ b/app/services/commits/cherry_pick_service.rb @@ -11,8 +11,6 @@ module Commits private def track_mr_picking(pick_sha) - return unless Feature.enabled?(:track_mr_picking, project) - merge_request = project.merge_requests.by_merge_commit_sha(@commit.sha).first return unless merge_request diff --git a/app/services/deployments/link_merge_requests_service.rb b/app/services/deployments/link_merge_requests_service.rb index 67a2230350d..eba5082e6c3 100644 --- a/app/services/deployments/link_merge_requests_service.rb +++ b/app/services/deployments/link_merge_requests_service.rb @@ -38,8 +38,6 @@ module Deployments .commits_between(from, to) .map(&:id) - track_mr_picking = Feature.enabled?(:track_mr_picking, project) - # For some projects the list of commits to deploy may be very large. To # ensure we do not end up running SQL queries with thousands of WHERE IN # values, we run one query per a certain number of commits. @@ -53,8 +51,6 @@ module Deployments deployment.link_merge_requests(merge_requests) - next unless track_mr_picking - picked_merge_requests = project.merge_requests.by_cherry_pick_sha(slice) diff --git a/app/services/system_notes/merge_requests_service.rb b/app/services/system_notes/merge_requests_service.rb index a26fc0f7d35..baf26245eb9 100644 --- a/app/services/system_notes/merge_requests_service.rb +++ b/app/services/system_notes/merge_requests_service.rb @@ -143,7 +143,7 @@ module SystemNotes def picked_into_branch(branch_name, pick_commit) link = url_helpers.project_tree_path(project, branch_name) - body = "picked this merge request into branch [`#{branch_name}`](#{link}) with commit #{pick_commit}" + body = "picked the changes into the branch [`#{branch_name}`](#{link}) with commit #{pick_commit}" summary = NoteSummary.new(noteable, project, author, body, action: 'cherry_pick') summary.note[:commit_id] = pick_commit diff --git a/app/views/shared/_no_ssh.html.haml b/app/views/shared/_no_ssh.html.haml index 17ef5327341..fbfd4d0e9a9 100644 --- a/app/views/shared/_no_ssh.html.haml +++ b/app/views/shared/_no_ssh.html.haml @@ -1,9 +1,10 @@ - if show_no_ssh_key_message? - .no-ssh-key-message.alert.alert-warning - - add_ssh_key_link = link_to s_('MissingSSHKeyWarningLink|add an SSH key'), profile_keys_path, class: 'alert-link' - - ssh_message = _("You won't be able to pull or push project code via SSH until you %{add_ssh_key_link} to your profile") % { add_ssh_key_link: add_ssh_key_link } - = ssh_message.html_safe - .alert-link-group - = link_to _("Don't show again"), profile_path(user: {hide_no_ssh_key: true}), method: :put, class: 'alert-link' - | - = link_to _('Remind later'), '#', class: 'hide-no-ssh-message alert-link' + %div{ class: 'no-ssh-key-message gl-alert gl-alert-warning', role: 'alert' } + = sprite_icon('warning', size: 16, css_class: 'gl-icon s16 gl-alert-icon gl-alert-icon-no-title') + %button{ class: 'gl-alert-dismiss hide-no-ssh-message', type: 'button', 'aria-label': 'Dismiss' } + = sprite_icon('close', size: 16, css_class: 'gl-icon s16') + .gl-alert-body + = s_("MissingSSHKeyWarningLink|You won't be able to pull or push project code via SSH until you add an SSH key to your profile").html_safe + .gl-alert-actions + = link_to s_('MissingSSHKeyWarningLink|Add SSH key'), profile_keys_path, class: "btn gl-alert-action btn-warning btn-md new-gl-button" + = link_to s_("MissingSSHKeyWarningLink|Don't show again"), profile_path(user: {hide_no_ssh_key: true}), method: :put, role: 'button', class: 'btn gl-alert-action btn-md btn-warning btn-secondary new-gl-button' diff --git a/changelogs/unreleased/208830-download-urls-conan-reference.yml b/changelogs/unreleased/208830-download-urls-conan-reference.yml new file mode 100644 index 00000000000..6b9732d514c --- /dev/null +++ b/changelogs/unreleased/208830-download-urls-conan-reference.yml @@ -0,0 +1,6 @@ +--- +title: Fix Conan package download_urls and snapshot to return files based on requested + conan_package_reference +merge_request: 27250 +author: +type: fixed diff --git a/changelogs/unreleased/208923-enable-batch-counting-for-some-individual-queries.yml b/changelogs/unreleased/208923-enable-batch-counting-for-some-individual-queries.yml new file mode 100644 index 00000000000..891ec3c0b58 --- /dev/null +++ b/changelogs/unreleased/208923-enable-batch-counting-for-some-individual-queries.yml @@ -0,0 +1,5 @@ +--- +title: Optimize clusters counters query performance in usage data +merge_request: 26887 +author: +type: performance diff --git a/changelogs/unreleased/26111-fix-github-gist-links.yml b/changelogs/unreleased/26111-fix-github-gist-links.yml new file mode 100644 index 00000000000..0f6954d2198 --- /dev/null +++ b/changelogs/unreleased/26111-fix-github-gist-links.yml @@ -0,0 +1,5 @@ +--- +title: Fix submodule links to gist.github.com +merge_request: 27346 +author: +type: fixed diff --git a/changelogs/unreleased/ac-remove-track_mr_picking-ff.yml b/changelogs/unreleased/ac-remove-track_mr_picking-ff.yml new file mode 100644 index 00000000000..b1ebe525f2f --- /dev/null +++ b/changelogs/unreleased/ac-remove-track_mr_picking-ff.yml @@ -0,0 +1,5 @@ +--- +title: Track merge request cherry-picks +merge_request: 26907 +author: +type: added diff --git a/changelogs/unreleased/creator-pairing-fix-alert.yml b/changelogs/unreleased/creator-pairing-fix-alert.yml new file mode 100644 index 00000000000..1b027d29d73 --- /dev/null +++ b/changelogs/unreleased/creator-pairing-fix-alert.yml @@ -0,0 +1,5 @@ +--- +title: Fixed SSH warning style +merge_request: 26992 +author: +type: other diff --git a/changelogs/unreleased/fix-x509-signed-commit.yml b/changelogs/unreleased/fix-x509-signed-commit.yml deleted file mode 100644 index d3d0f70ce15..00000000000 --- a/changelogs/unreleased/fix-x509-signed-commit.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix crl_url parsing and certificate visualization -merge_request: 25876 -author: Roger Meier -type: fixed diff --git a/changelogs/unreleased/unlink-cache-deletions.yml b/changelogs/unreleased/unlink-cache-deletions.yml new file mode 100644 index 00000000000..699ec94d1e1 --- /dev/null +++ b/changelogs/unreleased/unlink-cache-deletions.yml @@ -0,0 +1,5 @@ +--- +title: Swap to UNLINK for Redis set cache +merge_request: 27116 +author: +type: performance diff --git a/db/migrate/20200310123229_add_index_on_enabled_and_provider_type_and_id_to_clusters.rb b/db/migrate/20200310123229_add_index_on_enabled_and_provider_type_and_id_to_clusters.rb new file mode 100644 index 00000000000..edd9343e743 --- /dev/null +++ b/db/migrate/20200310123229_add_index_on_enabled_and_provider_type_and_id_to_clusters.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexOnEnabledAndProviderTypeAndIdToClusters < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :clusters, [:enabled, :provider_type, :id] + remove_concurrent_index :clusters, :enabled + end + + def down + add_concurrent_index :clusters, :enabled + remove_concurrent_index :clusters, [:enabled, :provider_type, :id] + end +end diff --git a/db/schema.rb b/db/schema.rb index 83aa4a32ffb..c089c4f123a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1125,7 +1125,7 @@ ActiveRecord::Schema.define(version: 2020_03_12_163407) do t.integer "management_project_id" t.integer "cleanup_status", limit: 2, default: 1, null: false t.text "cleanup_status_reason" - t.index ["enabled"], name: "index_clusters_on_enabled" + t.index ["enabled", "provider_type", "id"], name: "index_clusters_on_enabled_and_provider_type_and_id" t.index ["management_project_id"], name: "index_clusters_on_management_project_id", where: "(management_project_id IS NOT NULL)" t.index ["user_id"], name: "index_clusters_on_user_id" end diff --git a/doc/administration/lfs/manage_large_binaries_with_git_lfs.md b/doc/administration/lfs/manage_large_binaries_with_git_lfs.md index da6567348aa..61b8f28293f 100644 --- a/doc/administration/lfs/manage_large_binaries_with_git_lfs.md +++ b/doc/administration/lfs/manage_large_binaries_with_git_lfs.md @@ -252,7 +252,7 @@ on [Git Credential Storage documentation](https://git-scm.com/book/en/v2/Git-Too GitLab checks files to detect LFS pointers on push. If LFS pointers are detected, GitLab tries to verify that those files already exist in LFS on GitLab. -Verify that LFS in installed locally and consider a manual push with `git lfs push --all`. +Verify that LFS is installed locally and consider a manual push with `git lfs push --all`. If you are storing LFS files outside of GitLab you can disable LFS on the project by setting `lfs_enabled: false` with the [projects API](../../api/projects.md#edit-project). diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index b6caff283fc..f3c06a917a7 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -3934,6 +3934,11 @@ enum IssueState { opened } +""" +Represents untyped JSON +""" +scalar JSON + type Label { """ Background color of the label @@ -6061,6 +6066,31 @@ type Project { visibility: String """ + Vulnerabilities reported on the project. Available only when feature flag `first_class_vulnerabilities` is enabled. + """ + vulnerabilities( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): VulnerabilityConnection + + """ Web URL of the project """ webUrl: String @@ -8421,4 +8451,117 @@ enum VisibilityScopesEnum { internal private public +} + +""" +Represents a vulnerability. +""" +type Vulnerability { + """ + Description of the vulnerability + """ + description: String + + """ + GraphQL ID of the vulnerability + """ + id: ID! + + """ + The JSON location metadata for the vulnerability. Its format depends on the + type of the security scan that found the vulnerability + """ + location: JSON + + """ + Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST) + """ + reportType: VulnerabilityReportType + + """ + Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) + """ + severity: VulnerabilitySeverity + + """ + State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) + """ + state: VulnerabilityState + + """ + Title of the vulnerability + """ + title: String + + """ + URL to the vulnerability's details page + """ + vulnerabilityPath: String +} + +""" +The connection type for Vulnerability. +""" +type VulnerabilityConnection { + """ + A list of edges. + """ + edges: [VulnerabilityEdge] + + """ + A list of nodes. + """ + nodes: [Vulnerability] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type VulnerabilityEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: Vulnerability +} + +""" +The type of the security scan that found the vulnerability. +""" +enum VulnerabilityReportType { + CONTAINER_SCANNING + DAST + DEPENDENCY_SCANNING + SAST +} + +""" +The severity of the vulnerability. +""" +enum VulnerabilitySeverity { + CRITICAL + HIGH + INFO + LOW + MEDIUM + UNKNOWN +} + +""" +The state of the vulnerability. +""" +enum VulnerabilityState { + CONFIRMED + DETECTED + DISMISSED + RESOLVED }
\ No newline at end of file diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 38eae822983..65ff33c923c 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -11225,6 +11225,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "JSON", + "description": "Represents untyped JSON", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "Label", "description": null, @@ -18164,6 +18174,59 @@ "deprecationReason": null }, { + "name": "vulnerabilities", + "description": "Vulnerabilities reported on the project. Available only when feature flag `first_class_vulnerabilities` is enabled.", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the project", "args": [ @@ -25500,6 +25563,364 @@ }, { "kind": "OBJECT", + "name": "Vulnerability", + "description": "Represents a vulnerability.", + "fields": [ + { + "name": "description", + "description": "Description of the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "GraphQL ID of the vulnerability", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "location", + "description": "The JSON location metadata for the vulnerability. Its format depends on the type of the security scan that found the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "reportType", + "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST)", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "VulnerabilityReportType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "severity", + "description": "Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL)", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "VulnerabilitySeverity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED)", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "VulnerabilityState", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title of the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerabilityPath", + "description": "URL to the vulnerability's details page", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityConnection", + "description": "The connection type for Vulnerability.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Vulnerability", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "VulnerabilityReportType", + "description": "The type of the security scan that found the vulnerability.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "SAST", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DEPENDENCY_SCANNING", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CONTAINER_SCANNING", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DAST", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "VulnerabilitySeverity", + "description": "The severity of the vulnerability.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "INFO", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "UNKNOWN", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "LOW", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MEDIUM", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "HIGH", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CRITICAL", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "VulnerabilityState", + "description": "The state of the vulnerability.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "DETECTED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DISMISSED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "RESOLVED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CONFIRMED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "__Directive", "description": "A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document.\n\nIn some cases, you need to provide options to alter GraphQL's execution behavior in ways field arguments will not suffice, such as conditionally including or skipping a field. Directives provide this by describing additional information to the executor.", "fields": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a85553ce4aa..6d78f897017 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -1354,3 +1354,18 @@ Autogenerated return type of UpdateSnippet | Name | Type | Description | | --- | ---- | ---------- | | `createSnippet` | Boolean! | Indicates the user can perform `create_snippet` on this resource | + +## Vulnerability + +Represents a vulnerability. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | Description of the vulnerability | +| `id` | ID! | GraphQL ID of the vulnerability | +| `location` | JSON | The JSON location metadata for the vulnerability. Its format depends on the type of the security scan that found the vulnerability | +| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST) | +| `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | +| `state` | VulnerabilityState | State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) | +| `title` | String | Title of the vulnerability | +| `vulnerabilityPath` | String | URL to the vulnerability's details page | diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index 873d90dcf79..770726f4563 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -1,5 +1,13 @@ # Style guides +## Editor/IDE styling standardization + +We use [EditorConfig](https://editorconfig.org/) to automatically apply certain styling +standards before files are saved locally. Most editors/IDEs will honor the `.editorconfig` +settings automatically by default. If your editor/IDE does not automatically support `.editorconfig`, +we suggest investigating to see if a plugin exists. For instance here is the +[plugin for vim](https://github.com/editorconfig/editorconfig-vim). + ## Pre-commit static analysis You're strongly advised to install diff --git a/doc/development/database_review.md b/doc/development/database_review.md index a624abc2929..0cd7e083e2a 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -124,6 +124,8 @@ the following preparations into account. - Follow the [guidelines on dropping columns](what_requires_downtime.md#dropping-columns). - Generally it's best practice (but not a hard rule) to remove indexes and foreign keys in a post-deployment migration. - Exceptions include removing indexes and foreign keys for small tables. +- If you're adding a composite index, another index might become redundant, so remove that in the same migration. + For example adding `index(column_A, column_B, column_C)` makes the indexes `index(column_A, column_B)` and `index(column_A)` redundant. ### How to review for database diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index d701af20ec3..b6f6faa6052 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -166,6 +166,10 @@ environment, you can use the [GitLab Development Kit (GDK)](https://gitlab.com/g Please refer to the instructions in the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa/README.md#how-can-i-use-it) and the section below. +### Running tests that require special setup + +Learn how to perform [tests that require special setup or consideration to run on your local environment](running_tests_that_require_special_setup.md). + ## How do I write tests? In order to write new tests, you first need to learn more about GitLab QA diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md new file mode 100644 index 00000000000..f360226d922 --- /dev/null +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -0,0 +1,50 @@ +# Running tests that require special setup + +## Jenkins spec + +The [`jenkins_build_status_spec`](https://gitlab.com/gitlab-org/gitlab/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) spins up a Jenkins instance in a docker container based on an image stored in the [GitLab-QA container registry](https://gitlab.com/gitlab-org/gitlab-qa/container_registry). +The docker image it uses is preconfigured with some base data and plugins. +The test then configures the GitLab plugin in Jenkins with a URL of the GitLab instance that will be used +to run the tests. Unfortunately, the GitLab Jenkins plugin does not accept ports so `http://localhost:3000` would +not be accepted. Therefore, this requires us to run GitLab on port 80 or inside a docker container. + +To start a docker container for GitLab based on the nightly image: + +```shell +docker run \ + --publish 80:80 \ + --name gitlab \ + --hostname localhost \ + gitlab/gitlab-ee:nightly +``` + +To run the tests from the `/qa` directory: + +```shell +CHROME_HEADLESS=false bin/qa Test::Instance::All http://localhost -- qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb +``` + +The test will automatically spinup a docker container for Jenkins and tear down once the test completes. + +However, if you need to run Jenkins manually outside of the tests, use this command: + +```shell +docker run \ + --hostname localhost \ + --name jenkins-server \ + --env JENKINS_HOME=jenkins_home \ + --publish 8080:8080 \ + registry.gitlab.com/gitlab-org/gitlab-qa/jenkins-gitlab:version1 +``` + +Jenkins will be available on `http://localhost:8080`. + +Admin username is `admin` and password is `password`. + +It is worth noting that this is not an orchestrated test. It is [tagged with the `:orchestrated` meta](https://gitlab.com/gitlab-org/gitlab/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb#L5) +only to prevent it from running in the pipelines for live environments such as Staging. + +### Troubleshooting + +If Jenkins docker container exits without providing any information in the logs, try increasing the memory used by +the Docker Engine. diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 3028d575050..8e97427e061 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -62,7 +62,7 @@ The scanning tools and vulnerabilities database are updated regularly. | Secure scanning tool | Vulnerabilities database updates | |:-------------------------------------------------------------|-------------------------------------------| | [Container Scanning](container_scanning/index.md) | Uses `clair`. The latest `clair-db` version is used for each job by running the [`latest` docker image tag](https://gitlab.com/gitlab-org/gitlab/blob/438a0a56dc0882f22bdd82e700554525f552d91b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L37). The `clair-db` database [is updated daily according to the author](https://github.com/arminc/clair-local-scan#clair-server-or-local). | -| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Rubygems), `retire.js` (for NPM packages), and `gemnasium` (GitLab's own tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [Gemnasium DB](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated at least once a week. | +| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Rubygems), `retire.js` (for NPM packages), and `gemnasium` (GitLab's own tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [Gemnasium DB](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated at least once a week. See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | | [Dynamic Application Security Testing (DAST)](dast/index.md) | The scanning engine is updated on a periodic basis. See the [version of the underlying tool `zaproxy`](https://gitlab.com/gitlab-org/security-products/dast/blob/master/Dockerfile#L1). The scanning rules are downloaded at scan runtime. | | [Static Application Security Testing (SAST)](sast/index.md) | Relies exclusively on [the tools GitLab wraps](sast/index.md#supported-languages-and-frameworks). The underlying analyzers are updated at least once per month if a relevant update is available. The vulnerabilities database is updated by the upstream tools. | @@ -240,6 +240,15 @@ An approval is optional when a license report: - Contains no software license violations. - Contains only new licenses that are `approved` or unknown. +## Working in an offline environment + +It is possible to run most of the GitLab security scanners when not +connected to the internet, in what is sometimes known as an offline, +limited connectivity, Local Area Network (LAN), Intranet, or "air-gap" +environment. + +Read how to [operate the Secure scanners in an offline environment](offline_deployments/index.md). + ## Outdated security reports > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4913) in GitLab 12.7. diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md new file mode 100644 index 00000000000..f72b632ff82 --- /dev/null +++ b/doc/user/application_security/offline_deployments/index.md @@ -0,0 +1,55 @@ +--- +type: reference, howto +--- + +# Offline deployments + +This document describes how to operate Secure scanners offline. + +## Overview + +It is possible to run most of the GitLab security scanners when not +connected to the internet, in what is sometimes known as an offline, +limited connectivity, Local Area Network (LAN), Intranet, or "air-gap" +environment. + +In this situation, the GitLab instance can be one, or more, servers and services running in a network that can talk to one another, but have zero, or perhaps very restricted access to the internet. Assume anything within the GitLab instance and supporting infrastrusture (private maven repository for example) can be accessed via local network connection. Assume any files from the internet must come in via physical media (USB drive, hard drive). + +GitLab scanners generally will connect to the internet to download the +latest sets of signatures, rules, and patches. A few extra steps are necessary +to configure the tools to not do this and to still function properly. + +### Container registries and package repositories + +At a high-level, each of the security analyzers are delivered as Docker +containers and reference various package repositories. When you run a job on +an internet-connected GitLab installation, GitLab checks the GitLab.com-hosted +container registry and package repositories to ensure that you have +the latest versions. + +In an air-gapped environment, this must be disabled so that GitLab.com is not +queried. Because the GitLab.com registry and repositories are not available, +you must update each of the scanners to either reference a different, +internally-hosted registry or provide access to the individual scanner images. + +You must also ensure that your app has access to common package repos +that are not hosted on GitLab.com, such as npm, yarn, or rubygems. Packages +from these repos can be obtained by temporarily connecting to a network or by +mirroring the packages inside your own offline network. + +### Scanner signature and rule updates + +When connected to the internet, some scanners will reference public databases +for the latest sets of signatures and rules to check against. Without connectivity, +this is not possible. Depending on the scanner, you must therefore disable +these automatic update checks and either use the databases that they came +with or manually update those databases. + +## Specific scanner instructions + +Each individual scanner may be slightly different than the steps described +above. You can find more info at each of the pages below: + +- [Container scanning offline directions](../container_scanning/index.md#running-container-scanning-in-an-offline-air-gapped-installation) +- [SAST offline directions](../sast/index.md#gitlab-sast-in-an-offline-air-gapped-installation) +- [DAST offline directions](../dast/index.md#running-dast-in-an-offline-air-gapped-installation) diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 192941ff6e1..b342128e069 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -297,6 +297,25 @@ Ingress with the recent changes.  +##### Viewing Web Application Firewall traffic + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. + +You can view Web Application Firewall traffic by navigating to your project's +**Security & Compliance > Threat Monitoring** page. + +From there, you can see tracked over time: + +- The total amount of traffic to your application. +- The proportion of traffic that is considered anomalous by the Web Application + Firewall's default [OWASP ruleset](https://www.modsecurity.org/CRS/Documentation/). + +If a significant percentage of traffic is anomalous, it should be investigated +for potential threats, which can be done by +[examining the application logs](#web-application-firewall-modsecurity). + + + ### JupyterHub > - Introduced in GitLab 11.0 for project-level clusters. diff --git a/doc/user/clusters/img/threat_monitoring_v12_9.png b/doc/user/clusters/img/threat_monitoring_v12_9.png Binary files differnew file mode 100644 index 00000000000..9097f9334a8 --- /dev/null +++ b/doc/user/clusters/img/threat_monitoring_v12_9.png diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 361efdcba73..b81426d9089 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -88,11 +88,14 @@ or over the size limit, you can [reduce your repository size with Git](../projec ## IP range -GitLab.com, CI/CD, and related services are deployed into Google Cloud Platform (GCP). Any -IP based firewall can be configured by looking up all -[IP address ranges or CIDR blocks for GCP](https://cloud.google.com/compute/docs/faq#where_can_i_find_product_name_short_ip_ranges). +GitLab.com is using the IP range `34.74.90.64/28` for traffic from its Web/API +fleet. You can expect connections from webhooks or repository mirroring to come +from those IPs and whitelist them. -[Static endpoints](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/97) are being considered. +For connections from CI/CD runners we are not providing static IP addresses. +All our runners are deployed into Google Cloud Platform (GCP) - any IP based +firewall can be configured by looking up all +[IP address ranges or CIDR blocks for GCP](https://cloud.google.com/compute/docs/faq#where_can_i_find_product_name_short_ip_ranges). ## Maximum number of webhooks diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index 588c471bb64..2a624283e05 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -159,6 +159,9 @@ In most of the below cases, the notification will be sent to: - Subscribers: anyone who manually subscribed to the issue, merge request, or epic **(ULTIMATE)** - Custom: Users with notification level "custom" who turned on notifications for any of the events present in the table below +NOTE: **Note:** +To minimize the number of notifications that do not require any action, from [GitLab 12.9 onwards](https://gitlab.com/gitlab-org/gitlab/issues/616), eligible approvers are no longer notified for all the activities in their projects. To receive them they have to change their user notification settings to **Watch** instead. + | Event | Sent to | |------------------------|---------| | New issue | | diff --git a/doc/user/project/merge_requests/cherry_pick_changes.md b/doc/user/project/merge_requests/cherry_pick_changes.md index 1ba5ece89d5..3e76b9ec6b9 100644 --- a/doc/user/project/merge_requests/cherry_pick_changes.md +++ b/doc/user/project/merge_requests/cherry_pick_changes.md @@ -21,6 +21,20 @@ where you can choose to either: - Cherry-pick the changes directly into the selected branch. - Create a new merge request with the cherry-picked changes. +### Cherry-pick tracking + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2675) in GitLab 12.9. + +When you cherry-pick a merge commit, GitLab will output a system note to the related merge +request thread crosslinking the new commit and the existing merge request. + + + +Each deployment's [list of associated merge requests](../../../api/deployments.md#list-of-merge-requests-associated-with-a-deployment) will include cherry-picked merge commits. + +NOTE: **Note:** +We only track cherry-pick executed from GitLab (both UI and API). Support for [tracking cherry-picked commits through the command line](https://gitlab.com/gitlab-org/gitlab/issues/202215) is planned for a future release. + ## Cherry-picking a commit You can cherry-pick a commit from the commit details page: diff --git a/doc/user/project/merge_requests/img/cherry_pick_mr_timeline_v12_9.png b/doc/user/project/merge_requests/img/cherry_pick_mr_timeline_v12_9.png Binary files differnew file mode 100644 index 00000000000..919b576fcc6 --- /dev/null +++ b/doc/user/project/merge_requests/img/cherry_pick_mr_timeline_v12_9.png diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 359a9b4489a..f95ee26a226 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -297,7 +297,7 @@ For a list of known issues, visit GitLab's [public issue tracker]. [staticgen]: https://www.staticgen.com/ [pages-jekyll]: https://gitlab.com/pages/jekyll [metarefresh]: https://en.wikipedia.org/wiki/Meta_refresh -[public issue tracker]: https://gitlab.com/gitlab-org/gitlab-foss/issues?label_name=pages +[public issue tracker]: https://gitlab.com/gitlab-org/gitlab/-/issues?label_name[]=Category%3APages [quick start guide]: ../../../ci/quick_start/README.md [pages-index-guide]: index.md [pages-quick]: getting_started_part_one.md diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 50a68c2e2c4..c5174da4b7c 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -133,6 +133,7 @@ module Gitlab %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity, %r{\Atooling/overcommit/} => :engineering_productivity, + %r{\A.editorconfig\z} => :engineering_productivity, %r{Dangerfile\z} => :engineering_productivity, %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, %r{\A(ee/)?scripts/} => :engineering_productivity, diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 3940b3fca4b..e79127108b4 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -101,6 +101,8 @@ module Gitlab # redis.expire(key, EXPIRATION) end + + record_memory_usage(fetch_memory_usage(redis, key)) end # Subsequent read_file calls would need the latest cache. @@ -109,6 +111,23 @@ module Gitlab clear_memoization(:cacheable_files) end + def record_memory_usage(memory_usage) + if memory_usage + self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage) + end + end + + def fetch_memory_usage(redis, key) + # Redis versions prior to 4.0.0 do not support memory usage reporting + # for a specific key. As of 11-March-2020 we support Redis 3.x, so + # need to account for this. We can remove this check once we + # officially cease supporting versions <4.0.0. + # + return if Gem::Version.new(redis.info["redis_version"]) < Gem::Version.new("4") + + redis.memory("USAGE", key) + end + def file_paths strong_memoize(:file_paths) do diff_files.collect(&:file_path) diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 304f53b58c4..688a4a39dba 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -237,7 +237,7 @@ module Gitlab end def expire_redis_set_method_caches(methods) - methods.each { |name| redis_set_cache.expire(name) } + redis_set_cache.expire(*methods) end def expire_redis_hash_method_caches(methods) diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 0927a3e64bb..2e72855824a 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -14,8 +14,11 @@ module Gitlab "#{key}:set" end - def expire(key) - with { |redis| redis.del(cache_key(key)) } + def expire(*keys) + with do |redis| + keys = keys.map { |key| cache_key(key) } + unlink_or_delete(redis, keys) + end end def exist?(key) @@ -51,5 +54,15 @@ module Gitlab def with(&blk) Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end + + def unlink_or_delete(redis, keys) + if Feature.enabled?(:repository_set_cache_unlink, default_enabled: true) + redis.unlink(*keys) + else + redis.del(*keys) + end + rescue ::Redis::CommandError + redis.del(*keys) + end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6da238e61b9..b9cd4d74914 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -66,8 +66,8 @@ module Gitlab clusters_disabled: count(::Clusters::Cluster.disabled), project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), - clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled, batch: false), - clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled, batch: false), + clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_applications_helm: count(::Clusters::Applications::Helm.available), clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 837af28ba11..a335ede2ea9 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -363,6 +363,9 @@ msgstr "" msgid "%{name} found %{resultsString}" msgstr "" +msgid "%{name} is scheduled for %{action}" +msgstr "" + msgid "%{name}'s avatar" msgstr "" @@ -1585,6 +1588,9 @@ msgstr "" msgid "All" msgstr "" +msgid "All %{replicableType} are being scheduled for %{action}" +msgstr "" + msgid "All Members" msgstr "" @@ -12795,7 +12801,13 @@ msgstr "" msgid "Missing commit signatures endpoint!" msgstr "" -msgid "MissingSSHKeyWarningLink|add an SSH key" +msgid "MissingSSHKeyWarningLink|Add SSH key" +msgstr "" + +msgid "MissingSSHKeyWarningLink|Don't show again" +msgstr "" + +msgid "MissingSSHKeyWarningLink|You won't be able to pull or push project code via SSH until you add an SSH key to your profile" msgstr "" msgid "Modal|Cancel" @@ -13139,7 +13151,7 @@ msgstr "" msgid "No %{providerTitle} repositories found" msgstr "" -msgid "No Design Repositories match this filter" +msgid "No %{replicableType} match this filter" msgstr "" msgid "No Epic" @@ -16896,7 +16908,7 @@ msgstr "" msgid "Resync" msgstr "" -msgid "Resync all designs" +msgid "Resync all %{replicableType}" msgstr "" msgid "Retry" @@ -20013,10 +20025,10 @@ msgstr "" msgid "There was an error fetching median data for stages" msgstr "" -msgid "There was an error fetching the Node's Groups" +msgid "There was an error fetching the %{replicableType}" msgstr "" -msgid "There was an error fetching the designs" +msgid "There was an error fetching the Node's Groups" msgstr "" msgid "There was an error fetching the environments information." @@ -20061,7 +20073,10 @@ msgstr "" msgid "There was an error subscribing to this label." msgstr "" -msgid "There was an error syncing the designs." +msgid "There was an error syncing project %{name}" +msgstr "" + +msgid "There was an error syncing the %{replicableType}" msgstr "" msgid "There was an error trying to validate your query" @@ -23102,9 +23117,6 @@ msgstr "" msgid "You won't be able to pull or push project code via %{protocol} until you %{set_password_link} on your account" msgstr "" -msgid "You won't be able to pull or push project code via SSH until you %{add_ssh_key_link} to your profile" -msgstr "" - msgid "You won't be able to pull or push project code via SSH until you add an SSH key to your profile" msgstr "" diff --git a/qa/Dockerfile b/qa/Dockerfile index 126d9fbc591..155a9761a6f 100644 --- a/qa/Dockerfile +++ b/qa/Dockerfile @@ -1,5 +1,5 @@ FROM ruby:2.6-stretch -LABEL maintainer "Grzegorz Bizon <grzegorz@gitlab.com>" +LABEL maintainer="GitLab Quality Department <quality@gitlab.com>" ENV DEBIAN_FRONTEND noninteractive ## diff --git a/qa/README.md b/qa/README.md index cdc84da1c5e..c7c6e535963 100644 --- a/qa/README.md +++ b/qa/README.md @@ -44,6 +44,14 @@ Note: GitLab QA uses [Selenium WebDriver](https://www.seleniumhq.org/) via the browser to use. You will need to have Chrome (or Chromium) and [chromedriver](https://chromedriver.chromium.org/) installed / in your `$PATH`. +### Writing tests + +- [Writing tests from scratch tutorial](../doc/development/testing_guide/end_to_end/quick_start_guide.md) + - [Best practices](../doc/development/testing_guide/best_practices.md) + - [Using page objects](../doc/development/testing_guide/end_to_end/page_objects.md) + - [Guidelines](../doc/development/testing_guide/index.md) + - [Tests with special setup for local environemnts](../doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md) + ### Run the end-to-end tests in a local development environment Follow the GDK instructions to [prepare](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/prepare.md) @@ -77,13 +85,6 @@ Once you have the license file you can export it as an environment variable and export EE_LICENSE=$(cat /path/to/gitlab_license) ``` -### Writing tests - -- [Writing tests from scratch tutorial](../doc/development/testing_guide/end_to_end/quick_start_guide.md) - - [Best practices](../doc/development/testing_guide/best_practices.md) - - [Using page objects](../doc/development/testing_guide/end_to_end/page_objects.md) - - [Guidelines](../doc/development/testing_guide/index.md) - ### Running specific tests You can also supply specific tests to run as another parameter. For example, to diff --git a/spec/helpers/submodule_helper_spec.rb b/spec/helpers/submodule_helper_spec.rb index 7db45c05cb5..db0836c8550 100644 --- a/spec/helpers/submodule_helper_spec.rb +++ b/spec/helpers/submodule_helper_spec.rb @@ -81,6 +81,33 @@ describe SubmoduleHelper do end end + context 'submodule on gist.github.com' do + it 'detects ssh' do + stub_url('git@gist.github.com:gitlab-org/gitlab-foss.git') + is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash']) + end + + it 'detects http' do + stub_url('http://gist.github.com/gitlab-org/gitlab-foss.git') + is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash']) + end + + it 'detects https' do + stub_url('https://gist.github.com/gitlab-org/gitlab-foss.git') + is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash']) + end + + it 'handles urls with no .git on the end' do + stub_url('http://gist.github.com/gitlab-org/gitlab-foss') + is_expected.to eq(['https://gist.github.com/gitlab-org/gitlab-foss', 'https://gist.github.com/gitlab-org/gitlab-foss/hash']) + end + + it 'returns original with non-standard url' do + stub_url('http://gist.github.com/another/gitlab-org/gitlab-foss.git') + is_expected.to eq([repo.submodule_url_for, nil]) + end + end + context 'submodule on github.com' do it 'detects ssh' do stub_url('git@github.com:gitlab-org/gitlab-foss.git') diff --git a/spec/lib/gitlab/danger/helper_spec.rb b/spec/lib/gitlab/danger/helper_spec.rb index c76a1ffaa14..dd183281977 100644 --- a/spec/lib/gitlab/danger/helper_spec.rb +++ b/spec/lib/gitlab/danger/helper_spec.rb @@ -222,6 +222,7 @@ describe Gitlab::Danger::Helper do 'lib/gitlab/danger/foo' | :engineering_productivity 'ee/lib/gitlab/danger/foo' | :engineering_productivity '.overcommit.yml.example' | :engineering_productivity + '.editorconfig' | :engineering_productivity 'tooling/overcommit/foo' | :engineering_productivity 'lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml' | :backend diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb index 457e81bc95f..a16e5e185bb 100644 --- a/spec/lib/gitlab/diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb @@ -97,6 +97,28 @@ describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do let(:paths) { merge_request.diffs.raw_diff_files.select(&:text?).map(&:file_path) } end + it 'updates memory usage metrics if Redis version >= 4' do + allow_next_instance_of(Redis) do |redis| + allow(redis).to receive(:info).and_return({ "redis_version" => "4.0.0" }) + + expect(described_class.gitlab_redis_diff_caching_memory_usage_bytes) + .to receive(:observe).and_call_original + + cache.send(:write_to_redis_hash, diff_hash) + end + end + + it 'does not update memory usage metrics if Redis version < 4' do + allow_next_instance_of(Redis) do |redis| + allow(redis).to receive(:info).and_return({ "redis_version" => "3.0.0" }) + + expect(described_class.gitlab_redis_diff_caching_memory_usage_bytes) + .not_to receive(:observe).and_call_original + + cache.send(:write_to_redis_hash, diff_hash) + end + end + context 'different diff_collections for the same diffable' do before do cache.write_if_empty diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb index b4fc504ea60..dba5ffc84c5 100644 --- a/spec/lib/gitlab/repository_cache_adapter_spec.rb +++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb @@ -211,8 +211,7 @@ describe Gitlab::RepositoryCacheAdapter do it 'expires the caches of the given methods' do expect(cache).to receive(:expire).with(:rendered_readme) expect(cache).to receive(:expire).with(:branch_names) - expect(redis_set_cache).to receive(:expire).with(:rendered_readme) - expect(redis_set_cache).to receive(:expire).with(:branch_names) + expect(redis_set_cache).to receive(:expire).with(:rendered_readme, :branch_names) expect(redis_hash_cache).to receive(:delete).with(:rendered_readme) expect(redis_hash_cache).to receive(:delete).with(:branch_names) diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb index bcf27b338f6..c5d95e53120 100644 --- a/spec/lib/gitlab/repository_set_cache_spec.rb +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -51,12 +51,52 @@ describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do end describe '#expire' do - it 'expires the given key from the cache' do + subject { cache.expire(*keys) } + + before do cache.write(:foo, ['value']) + cache.write(:bar, ['value2']) + end + it 'actually wrote the values' do expect(cache.read(:foo)).to contain_exactly('value') - expect(cache.expire(:foo)).to eq(1) - expect(cache.read(:foo)).to be_empty + expect(cache.read(:bar)).to contain_exactly('value2') + end + + context 'single key' do + let(:keys) { %w(foo) } + + it { is_expected.to eq(1) } + + it 'deletes the given key from the cache' do + subject + + expect(cache.read(:foo)).to be_empty + end + end + + context 'multiple keys' do + let(:keys) { %w(foo bar) } + + it { is_expected.to eq(2) } + + it 'deletes the given keys from the cache' do + subject + + expect(cache.read(:foo)).to be_empty + expect(cache.read(:bar)).to be_empty + end + end + + context "unlink isn't supported" do + before do + allow_any_instance_of(Redis).to receive(:unlink) { raise ::Redis::CommandError } + end + + it 'still deletes the given key' do + expect(cache.expire(:foo)).to eq(1) + expect(cache.read(:foo)).to be_empty + end end end diff --git a/spec/services/commits/cherry_pick_service_spec.rb b/spec/services/commits/cherry_pick_service_spec.rb index ead1932c2d1..3b797b8ac02 100644 --- a/spec/services/commits/cherry_pick_service_spec.rb +++ b/spec/services/commits/cherry_pick_service_spec.rb @@ -61,18 +61,6 @@ describe Commits::CherryPickService do expect(mr_notes.length).to eq(1) expect(mr_notes[0].commit_id).to eq(result[:result]) end - - context 'when :track_mr_picking feature flag is disabled' do - before do - stub_feature_flags(track_mr_picking: false) - end - - it 'does not add system notes' do - expect do - cherry_pick(merge_commit_sha, branch_name) - end.not_to change { Note.count } - end - end end def find_cherry_pick_notes(noteable) diff --git a/spec/services/deployments/link_merge_requests_service_spec.rb b/spec/services/deployments/link_merge_requests_service_spec.rb index 2f635c7ad62..aa2cecbf897 100644 --- a/spec/services/deployments/link_merge_requests_service_spec.rb +++ b/spec/services/deployments/link_merge_requests_service_spec.rb @@ -160,53 +160,6 @@ describe Deployments::LinkMergeRequestsService do expect(deploy.merge_requests).to be_empty end - - context 'when :track_mr_picking feature flag is disabled' do - before do - stub_feature_flags(track_mr_picking: false) - end - - it 'does not link picked merge requests' do - environment = create(:environment, project: project) - deploy = - create(:deployment, :success, project: project, environment: environment) - - picked_mr = create( - :merge_request, - :merged, - merge_commit_sha: '123abc', - source_project: project, - target_project: project - ) - - mr1 = create( - :merge_request, - :merged, - merge_commit_sha: mr1_merge_commit_sha, - source_project: project, - target_project: project - ) - - # mr1 includes c1c67abba which is a cherry-pick of the fake picked_mr merge request - create(:track_mr_picking_note, noteable: picked_mr, project: project, commit_id: 'c1c67abbaf91f624347bb3ae96eabe3a1b742478') - - mr2 = create( - :merge_request, - :merged, - merge_commit_sha: mr2_merge_commit_sha, - source_project: project, - target_project: project - ) - - described_class.new(deploy).link_merge_requests_for_range( - first_deployment_sha, - mr2_merge_commit_sha - ) - - expect(deploy.merge_requests).to include(mr1, mr2) - expect(deploy.merge_requests).not_to include(picked_mr) - end - end end describe '#link_all_merged_merge_requests' do diff --git a/spec/services/system_notes/merge_requests_service_spec.rb b/spec/services/system_notes/merge_requests_service_spec.rb index f5c071502f5..13d6367a585 100644 --- a/spec/services/system_notes/merge_requests_service_spec.rb +++ b/spec/services/system_notes/merge_requests_service_spec.rb @@ -253,7 +253,7 @@ describe ::SystemNotes::MergeRequestsService do end it "posts the 'picked merge request' system note" do - expect(subject.note).to eq("picked this merge request into branch [`#{branch_name}`](/#{project.full_path}/-/tree/#{branch_name}) with commit #{commit_sha}") + expect(subject.note).to eq("picked the changes into the branch [`#{branch_name}`](/#{project.full_path}/-/tree/#{branch_name}) with commit #{commit_sha}") end it 'links the merge request and the cherry-pick commit' do |
